Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove all top-level imports for twisted.internet.reactor (#4406)
Aditya Kumarf9bf4b8d4dd6
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex a70023efa..c01202a10 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1473,7 +1473,66 @@ If a reactor is already installed,:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises:exc:`Exception` if the installed reactor does not match the-:setting:`TWISTED_REACTOR` setting.+:setting:`TWISTED_REACTOR` setting; therfore, having top-level+:mod:`~twisted.internet.reactor` imports in project files and imported+third-party libraries will make Scrapy raise :exc:`Exception` when+it checks which reactor is installed.++In order to use the reactor installed by Scrapy::++ import scrapy+ from twisted.internet import reactor+++ class QuotesSpider(scrapy.Spider):+ name = 'quotes'++ def __init__(self, *args, **kwargs):+ self.timeout = int(kwargs.pop('timeout', '60'))+ super(QuotesSpider, self).__init__(*args, **kwargs)++ def start_requests(self):+ reactor.callLater(self.timeout, self.stop)++ urls = ['http://quotes.toscrape.com/page/1']+ for url in urls:+ yield scrapy.Request(url=url, callback=self.parse)++ def parse(self, response):+ for quote in response.css('div.quote'):+ yield {'text': quote.css('span.text::text').get()}++ def stop(self):+ self.crawler.engine.close_spider(self, 'timeout')+++which raises :exc:`Exception`, becomes::++ import scrapy+++ class QuotesSpider(scrapy.Spider):+ name = 'quotes'++ def __init__(self, *args, **kwargs):+ self.timeout = int(kwargs.pop('timeout', '60'))+ super(QuotesSpider, self).__init__(*args, **kwargs)++ def start_requests(self):+ from twisted.internet import reactor+ reactor.callLater(self.timeout, self.stop)++ urls = ['http://quotes.toscrape.com/page/1']+ for url in urls:+ yield scrapy.Request(url=url, callback=self.parse)++ def parse(self, response):+ for quote in response.css('div.quote'):+ yield {'text': quote.css('span.text::text').get()}++ def stop(self):+ self.crawler.engine.close_spider(self, 'timeout')+The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, whichmeans that Scrapy will not attempt to install any specific reactor, and thediff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 5a2fdadf5..644be121f 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -3,7 +3,7 @@ from time import timefrom datetime import datetimefrom collections import deque-from twisted.internet import reactor, defer, task+from twisted.internet import defer, taskfrom scrapy.utils.defer import mustbe_deferredfrom scrapy.utils.httpobj import urlparse_cached@@ -133,6 +133,7 @@ class Downloader(object):return deferreddef _process_queue(self, spider, slot):+ from twisted.internet import reactorif slot.latercall and slot.latercall.active():returndiff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 1681c6df8..432cb1831 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -32,7 +32,6 @@ import refrom io import BytesIOfrom urllib.parse import unquote-from twisted.internet import reactorfrom twisted.internet.protocol import ClientCreator, Protocolfrom twisted.protocols.ftp import CommandFailed, FTPClient@@ -81,6 +80,7 @@ class FTPDownloadHandler:return cls(crawler.settings)def download_request(self, request, spider):+ from twisted.internet import reactorparsed_url = urlparse_cached(request)user = request.meta.get("ftp_user", self.default_user)password = request.meta.get("ftp_password", self.default_password)diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.pyindex d4aa51bd1..c0146a0a6 100644--- a/scrapy/core/downloader/handlers/http10.py+++ b/scrapy/core/downloader/handlers/http10.py@@ -1,7 +1,5 @@"""Download handlers for http and https schemes"""-from twisted.internet import reactor-from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.python import to_unicode@@ -26,6 +24,7 @@ class HTTP10DownloadHandler:return factory.deferreddef _connect(self, factory):+ from twisted.internet import reactorhost, port = to_unicode(factory.host), factory.portif factory.scheme == b'https':client_context_factory = create_instance(diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 93951d3b5..04a8d617a 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -8,7 +8,7 @@ from io import BytesIOfrom time import timefrom urllib.parse import urldefrag-from twisted.internet import defer, protocol, reactor, ssl+from twisted.internet import defer, protocol, sslfrom twisted.internet.endpoints import TCP4ClientEndpointfrom twisted.internet.error import TimeoutErrorfrom twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI@@ -33,6 +33,7 @@ class HTTP11DownloadHandler:lazy = Falsedef __init__(self, settings, crawler=None):+ from twisted.internet import reactorself._pool = HTTPConnectionPool(reactor, persistent=True)self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')self._pool._factory.noisy = False@@ -81,6 +82,7 @@ class HTTP11DownloadHandler:return agent.download_request(request)def close(self):+ from twisted.internet import reactord = self._pool.closeCachedConnections()# closeCachedConnections will hang on network or server issues, so# we'll manually timeout the deferred.@@ -284,6 +286,7 @@ class ScrapyAgent(object):self._txresponse = Nonedef _get_agent(self, request, timeout):+ from twisted.internet import reactorbindaddress = request.meta.get('bindaddress') or self._bindAddressproxy = request.meta.get('proxy')if proxy:@@ -326,6 +329,7 @@ class ScrapyAgent(object):)def download_request(self, request):+ from twisted.internet import reactortimeout = request.meta.get('download_timeout') or self._connectTimeoutagent = self._get_agent(request, timeout)diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.pyindex afb2ed049..260b2e86e 100644--- a/scrapy/extensions/closespider.py+++ b/scrapy/extensions/closespider.py@@ -6,8 +6,6 @@ See documentation in docs/topics/extensions.rstfrom collections import defaultdict-from twisted.internet import reactor-from scrapy import signalsfrom scrapy.exceptions import NotConfigured@@ -54,6 +52,7 @@ class CloseSpider(object):self.crawler.engine.close_spider(spider, 'closespider_pagecount')def spider_opened(self, spider):+ from twisted.internet import reactorself.task = reactor.callLater(self.close_on['timeout'],self.crawler.engine.close_spider, spider,reason='closespider_timeout')diff --git a/scrapy/mail.py b/scrapy/mail.pyindex 9655b8114..b2a24a3db 100644--- a/scrapy/mail.py+++ b/scrapy/mail.py@@ -12,7 +12,7 @@ from email.mime.text import MIMETextfrom email.utils import COMMASPACE, formatdatefrom io import BytesIO-from twisted.internet import defer, reactor, ssl+from twisted.internet import defer, sslfrom scrapy.utils.misc import arg_to_iterfrom scrapy.utils.python import to_bytes@@ -28,7 +28,6 @@ def _to_bytes_or_none(text):class MailSender(object):-def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):self.smtphost = smtphost@@ -47,6 +46,7 @@ class MailSender(object):settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):+ from twisted.internet import reactorif attachs:msg = MIMEMultipart()else:@@ -111,6 +111,7 @@ class MailSender(object):def _sendmail(self, to_addrs, msg):# Import twisted.mail here because it is not available in python3+ from twisted.internet import reactorfrom twisted.mail.smtp import ESMTPSenderFactorymsg = BytesIO(msg)d = defer.Deferred()diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.pyindex cdbe21942..9d8d64612 100644--- a/scrapy/utils/benchserver.py+++ b/scrapy/utils/benchserver.py@@ -3,7 +3,6 @@ from urllib.parse import urlencodefrom twisted.web.server import Sitefrom twisted.web.resource import Resource-from twisted.internet import reactorclass Root(Resource):@@ -34,6 +33,7 @@ def _getarg(request, name, default=None, type=str):if __name__ == '__main__':+ from twisted.internet import reactorroot = Root()factory = Site(root)httpPort = reactor.listenTCP(8998, Site(root))diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.pyindex 0f15cf60a..37803b287 100644--- a/scrapy/utils/testproc.py+++ b/scrapy/utils/testproc.py@@ -1,7 +1,7 @@import sysimport os-from twisted.internet import reactor, defer, protocol+from twisted.internet import defer, protocolclass ProcessTest(object):@@ -11,6 +11,7 @@ class ProcessTest(object):cwd = os.getcwd() # trial chdirs to temp dirdef execute(self, args, check_code=True, settings=None):+ from twisted.internet import reactorenv = os.environ.copy()if settings is not None:env['SCRAPY_SETTINGS_MODULE'] = settingsdiff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.pyindex 6f5c21624..9e1598805 100644--- a/scrapy/utils/testsite.py+++ b/scrapy/utils/testsite.py@@ -1,12 +1,12 @@from urllib.parse import urljoin-from twisted.internet import reactorfrom twisted.web import server, resource, static, utilclass SiteTest(object):def setUp(self):+ from twisted.internet import reactorsuper(SiteTest, self).setUp()self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")self.baseurl = "http://localhost:%d/" % self.site.getHost().port@@ -38,6 +38,7 @@ def test_site():if __name__ == '__main__':+ from twisted.internet import reactorport = reactor.listenTCP(0, test_site(), interface="127.0.0.1")print("http://localhost:%d/" % port.getHost().port)reactor.run()