Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove all top-level imports for twisted.internet.reactor (#4406)

Aditya Kumarf9bf4b8d4dd6
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a70023efa..c01202a10 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1473,7 +1473,66 @@ If a reactor is already installed,
:meth:`CrawlerRunner.__init__ <scrapy.crawler.CrawlerRunner.__init__>` raises
:exc:`Exception` if the installed reactor does not match the
-:setting:`TWISTED_REACTOR` setting.
+:setting:`TWISTED_REACTOR` setting; therfore, having top-level
+:mod:`~twisted.internet.reactor` imports in project files and imported
+third-party libraries will make Scrapy raise :exc:`Exception` when
+it checks which reactor is installed.
+
+In order to use the reactor installed by Scrapy::
+
+ import scrapy
+ from twisted.internet import reactor
+
+
+ class QuotesSpider(scrapy.Spider):
+ name = 'quotes'
+
+ def __init__(self, *args, **kwargs):
+ self.timeout = int(kwargs.pop('timeout', '60'))
+ super(QuotesSpider, self).__init__(*args, **kwargs)
+
+ def start_requests(self):
+ reactor.callLater(self.timeout, self.stop)
+
+ urls = ['http://quotes.toscrape.com/page/1']
+ for url in urls:
+ yield scrapy.Request(url=url, callback=self.parse)
+
+ def parse(self, response):
+ for quote in response.css('div.quote'):
+ yield {'text': quote.css('span.text::text').get()}
+
+ def stop(self):
+ self.crawler.engine.close_spider(self, 'timeout')
+
+
+which raises :exc:`Exception`, becomes::
+
+ import scrapy
+
+
+ class QuotesSpider(scrapy.Spider):
+ name = 'quotes'
+
+ def __init__(self, *args, **kwargs):
+ self.timeout = int(kwargs.pop('timeout', '60'))
+ super(QuotesSpider, self).__init__(*args, **kwargs)
+
+ def start_requests(self):
+ from twisted.internet import reactor
+ reactor.callLater(self.timeout, self.stop)
+
+ urls = ['http://quotes.toscrape.com/page/1']
+ for url in urls:
+ yield scrapy.Request(url=url, callback=self.parse)
+
+ def parse(self, response):
+ for quote in response.css('div.quote'):
+ yield {'text': quote.css('span.text::text').get()}
+
+ def stop(self):
+ self.crawler.engine.close_spider(self, 'timeout')
+
The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
means that Scrapy will not attempt to install any specific reactor, and the
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 5a2fdadf5..644be121f 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -3,7 +3,7 @@ from time import time
from datetime import datetime
from collections import deque
-from twisted.internet import reactor, defer, task
+from twisted.internet import defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
@@ -133,6 +133,7 @@ class Downloader(object):
return deferred
def _process_queue(self, spider, slot):
+ from twisted.internet import reactor
if slot.latercall and slot.latercall.active():
return
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 1681c6df8..432cb1831 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -32,7 +32,6 @@ import re
from io import BytesIO
from urllib.parse import unquote
-from twisted.internet import reactor
from twisted.internet.protocol import ClientCreator, Protocol
from twisted.protocols.ftp import CommandFailed, FTPClient
@@ -81,6 +80,7 @@ class FTPDownloadHandler:
return cls(crawler.settings)
def download_request(self, request, spider):
+ from twisted.internet import reactor
parsed_url = urlparse_cached(request)
user = request.meta.get("ftp_user", self.default_user)
password = request.meta.get("ftp_password", self.default_password)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index d4aa51bd1..c0146a0a6 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,7 +1,5 @@
"""Download handlers for http and https schemes
"""
-from twisted.internet import reactor
-
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import to_unicode
@@ -26,6 +24,7 @@ class HTTP10DownloadHandler:
return factory.deferred
def _connect(self, factory):
+ from twisted.internet import reactor
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
client_context_factory = create_instance(
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 93951d3b5..04a8d617a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,7 +8,7 @@ from io import BytesIO
from time import time
from urllib.parse import urldefrag
-from twisted.internet import defer, protocol, reactor, ssl
+from twisted.internet import defer, protocol, ssl
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
@@ -33,6 +33,7 @@ class HTTP11DownloadHandler:
lazy = False
def __init__(self, settings, crawler=None):
+ from twisted.internet import reactor
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
@@ -81,6 +82,7 @@ class HTTP11DownloadHandler:
return agent.download_request(request)
def close(self):
+ from twisted.internet import reactor
d = self._pool.closeCachedConnections()
# closeCachedConnections will hang on network or server issues, so
# we'll manually timeout the deferred.
@@ -284,6 +286,7 @@ class ScrapyAgent(object):
self._txresponse = None
def _get_agent(self, request, timeout):
+ from twisted.internet import reactor
bindaddress = request.meta.get('bindaddress') or self._bindAddress
proxy = request.meta.get('proxy')
if proxy:
@@ -326,6 +329,7 @@ class ScrapyAgent(object):
)
def download_request(self, request):
+ from twisted.internet import reactor
timeout = request.meta.get('download_timeout') or self._connectTimeout
agent = self._get_agent(request, timeout)
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index afb2ed049..260b2e86e 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -6,8 +6,6 @@ See documentation in docs/topics/extensions.rst
from collections import defaultdict
-from twisted.internet import reactor
-
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -54,6 +52,7 @@ class CloseSpider(object):
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
+ from twisted.internet import reactor
self.task = reactor.callLater(self.close_on['timeout'],
self.crawler.engine.close_spider, spider,
reason='closespider_timeout')
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 9655b8114..b2a24a3db 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -12,7 +12,7 @@ from email.mime.text import MIMEText
from email.utils import COMMASPACE, formatdate
from io import BytesIO
-from twisted.internet import defer, reactor, ssl
+from twisted.internet import defer, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
@@ -28,7 +28,6 @@ def _to_bytes_or_none(text):
class MailSender(object):
-
def __init__(self, smtphost='localhost', mailfrom='scrapy@localhost',
smtpuser=None, smtppass=None, smtpport=25, smtptls=False, smtpssl=False, debug=False):
self.smtphost = smtphost
@@ -47,6 +46,7 @@ class MailSender(object):
settings.getbool('MAIL_TLS'), settings.getbool('MAIL_SSL'))
def send(self, to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None, _callback=None):
+ from twisted.internet import reactor
if attachs:
msg = MIMEMultipart()
else:
@@ -111,6 +111,7 @@ class MailSender(object):
def _sendmail(self, to_addrs, msg):
# Import twisted.mail here because it is not available in python3
+ from twisted.internet import reactor
from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index cdbe21942..9d8d64612 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -3,7 +3,6 @@ from urllib.parse import urlencode
from twisted.web.server import Site
from twisted.web.resource import Resource
-from twisted.internet import reactor
class Root(Resource):
@@ -34,6 +33,7 @@ def _getarg(request, name, default=None, type=str):
if __name__ == '__main__':
+ from twisted.internet import reactor
root = Root()
factory = Site(root)
httpPort = reactor.listenTCP(8998, Site(root))
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 0f15cf60a..37803b287 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,7 +1,7 @@
import sys
import os
-from twisted.internet import reactor, defer, protocol
+from twisted.internet import defer, protocol
class ProcessTest(object):
@@ -11,6 +11,7 @@ class ProcessTest(object):
cwd = os.getcwd() # trial chdirs to temp dir
def execute(self, args, check_code=True, settings=None):
+ from twisted.internet import reactor
env = os.environ.copy()
if settings is not None:
env['SCRAPY_SETTINGS_MODULE'] = settings
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index 6f5c21624..9e1598805 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,12 +1,12 @@
from urllib.parse import urljoin
-from twisted.internet import reactor
from twisted.web import server, resource, static, util
class SiteTest(object):
def setUp(self):
+ from twisted.internet import reactor
super(SiteTest, self).setUp()
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
self.baseurl = "http://localhost:%d/" % self.site.getHost().port
@@ -38,6 +38,7 @@ def test_site():
if __name__ == '__main__':
+ from twisted.internet import reactor
port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
print("http://localhost:%d/" % port.getHost().port)
reactor.run()