Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Download handlers: from_crawler factory method, take crawler instead of settings in __init__

Eugenio Lacuesta3d77f74e4089
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..e8beb2f5a 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -5,7 +5,7 @@ from twisted.internet import defer
import six
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
from scrapy import signals
@@ -48,7 +48,11 @@ class DownloadHandlers(object):
dhcls = load_object(path)
if skip_lazy and getattr(dhcls, 'lazy', True):
return None
- dh = dhcls(self._crawler.settings)
+ dh = create_instance(
+ dhcls,
+ self._crawler.settings,
+ self._crawler,
+ )
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)
return None
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index 9e5020753..97134e618 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -8,9 +8,6 @@ from scrapy.utils.decorators import defers
class DataURIDownloadHandler(object):
lazy = False
- def __init__(self, settings):
- super(DataURIDownloadHandler, self).__init__()
-
@defers
def download_request(self, request, spider):
uri = parse_data_uri(request.url)
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 23f25d28d..d445ba2e1 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,4 +1,5 @@
from w3lib.url import file_uri_to_path
+
from scrapy.responsetypes import responsetypes
from scrapy.utils.decorators import defers
@@ -6,9 +7,6 @@ from scrapy.utils.decorators import defers
class FileDownloadHandler(object):
lazy = False
- def __init__(self, settings):
- pass
-
@defers
def download_request(self, request, spider):
filepath = file_uri_to_path(request.url)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 39ed67a1a..7a98361ed 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -59,6 +59,7 @@ class ReceivedDataProtocol(Protocol):
def close(self):
self.body.close() if self.filename else self.body.seek(0)
+
_CODE_RE = re.compile(r"\d+")
@@ -70,10 +71,14 @@ class FTPDownloadHandler(object):
"default": 503,
}
- def __init__(self, settings):
- self.default_user = settings['FTP_USER']
- self.default_password = settings['FTP_PASSWORD']
- self.passive_mode = settings['FTP_PASSIVE_MODE']
+ def __init__(self, crawler):
+ self.default_user = crawler.settings['FTP_USER']
+ self.default_password = crawler.settings['FTP_PASSWORD']
+ self.passive_mode = crawler.settings['FTP_PASSIVE_MODE']
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
def download_request(self, request, spider):
parsed_url = urlparse_cached(request)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index be7298531..ce0801bcc 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,6 +1,7 @@
"""Download handlers for http and https schemes
"""
from twisted.internet import reactor
+
from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.python import to_unicode
@@ -8,10 +9,15 @@ from scrapy.utils.python import to_unicode
class HTTP10DownloadHandler(object):
lazy = False
- def __init__(self, settings):
- self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
- self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
- self._settings = settings
+ def __init__(self, crawler):
+ self.HTTPClientFactory = load_object(crawler.settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+ self.ClientContextFactory = load_object(crawler.settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
+ self._crawler = crawler
+ self._settings = crawler.settings
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
@@ -22,7 +28,11 @@ class HTTP10DownloadHandler(object):
def _connect(self, factory):
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
- client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
+ client_context_factory = create_instance(
+ self.ClientContextFactory,
+ settings=self._settings,
+ crawler=self._crawler,
+ )
return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..b424a7999 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -30,7 +30,9 @@ logger = logging.getLogger(__name__)
class HTTP11DownloadHandler(object):
lazy = False
- def __init__(self, settings):
+ def __init__(self, crawler):
+ settings = crawler.settings
+
self._pool = HTTPConnectionPool(reactor, persistent=True)
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
self._pool._factory.noisy = False
@@ -42,7 +44,7 @@ class HTTP11DownloadHandler(object):
self._contextFactory = create_instance(
self._contextFactoryClass,
settings=settings,
- crawler=None,
+ crawler=crawler,
method=self._sslMethod,
)
except TypeError:
@@ -50,7 +52,7 @@ class HTTP11DownloadHandler(object):
self._contextFactory = create_instance(
self._contextFactoryClass,
settings=settings,
- crawler=None,
+ crawler=crawler,
)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
@@ -63,6 +65,10 @@ class HTTP11DownloadHandler(object):
self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')
self._disconnect_timeout = 1
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
agent = ScrapyAgent(
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 808d1bf21..220296fb3 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -32,13 +32,12 @@ def _get_boto_connection():
class S3DownloadHandler(object):
- def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
- httpdownloadhandler=HTTPDownloadHandler, **kw):
-
+ def __init__(self, crawler, aws_access_key_id=None, aws_secret_access_key=None,
+ httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
- aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
+ aws_access_key_id = crawler.settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
- aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+ aws_secret_access_key = crawler.settings['AWS_SECRET_ACCESS_KEY']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
@@ -67,7 +66,11 @@ class S3DownloadHandler(object):
except Exception as ex:
raise NotConfigured(str(ex))
- self._download_http = httpdownloadhandler(settings).download_request
+ self._download_http = httpdownloadhandler(crawler).download_request
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
def download_request(self, request, spider):
p = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 60124b93f..815342219 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -30,7 +30,6 @@ from scrapy.spiders import Spider
from scrapy.http import Headers, Request
from scrapy.http.response.text import TextResponse
from scrapy.responsetypes import responsetypes
-from scrapy.settings import Settings
from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
from scrapy.exceptions import NotConfigured
@@ -45,6 +44,10 @@ class DummyDH(object):
def __init__(self, crawler):
pass
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
class DummyLazyDH(object):
# Default is lazy for backward compatibility
@@ -52,6 +55,10 @@ class DummyLazyDH(object):
def __init__(self, crawler):
pass
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
class OffDH(object):
lazy = False
@@ -59,6 +66,10 @@ class OffDH(object):
def __init__(self, crawler):
raise NotConfigured
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
class LoadTestCase(unittest.TestCase):
@@ -106,7 +117,7 @@ class FileTestCase(unittest.TestCase):
self.tmpname = self.mktemp()
with open(self.tmpname + '^', 'w') as f:
f.write('0123456789')
- self.download_request = FileDownloadHandler(Settings()).download_request
+ self.download_request = FileDownloadHandler().download_request
def tearDown(self):
os.unlink(self.tmpname + '^')
@@ -239,7 +250,7 @@ class HttpTestCase(unittest.TestCase):
else:
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
self.portno = self.port.getHost().port
- self.download_handler = self.download_handler_cls(Settings())
+ self.download_handler = self.download_handler_cls(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@@ -479,9 +490,9 @@ class Http11TestCase(HttpTestCase):
return self.test_download_broken_content_allow_data_loss('broken-chunked')
def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
- download_handler = self.download_handler_cls(Settings({
- 'DOWNLOAD_FAIL_ON_DATALOSS': False,
- }))
+ download_handler = self.download_handler_cls(
+ get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
+ )
request = Request(self.getURL(url))
d = download_handler.download_request(request, Spider('foo'))
d.addCallback(lambda r: r.flags)
@@ -499,9 +510,9 @@ class Https11TestCase(Http11TestCase):
@defer.inlineCallbacks
def test_tls_logging(self):
- download_handler = self.download_handler_cls(Settings({
- 'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,
- }))
+ download_handler = self.download_handler_cls(
+ get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
+ )
try:
with LogCapture() as log_capture:
request = Request(self.getURL('file'))
@@ -569,7 +580,8 @@ class Https11CustomCiphers(unittest.TestCase):
interface=self.host)
self.portno = self.port.getHost().port
self.download_handler = self.download_handler_cls(
- Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))
+ get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+ )
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@@ -665,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
- self.download_handler = self.download_handler_cls(Settings())
+ self.download_handler = self.download_handler_cls(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@@ -738,9 +750,10 @@ class S3AnonTestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
- self.s3reqh = S3DownloadHandler(Settings(),
- httpdownloadhandler=HttpDownloadHandlerMock,
- #anon=True, # is implicit
+ self.s3reqh = S3DownloadHandler(
+ crawler=get_crawler(),
+ httpdownloadhandler=HttpDownloadHandlerMock,
+ #anon=True, # is implicit
)
self.download_request = self.s3reqh.download_request
self.spider = Spider('foo')
@@ -766,9 +779,12 @@ class S3TestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
- s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
- self.AWS_SECRET_ACCESS_KEY,
- httpdownloadhandler=HttpDownloadHandlerMock)
+ s3reqh = S3DownloadHandler(
+ get_crawler(),
+ self.AWS_ACCESS_KEY_ID,
+ self.AWS_SECRET_ACCESS_KEY,
+ httpdownloadhandler=HttpDownloadHandlerMock,
+ )
self.download_request = s3reqh.download_request
self.spider = Spider('foo')
@@ -788,7 +804,7 @@ class S3TestCase(unittest.TestCase):
def test_extra_kw(self):
try:
- S3DownloadHandler(Settings(), extra_kw=True)
+ S3DownloadHandler(get_crawler(), extra_kw=True)
except Exception as e:
self.assertIsInstance(e, (TypeError, NotConfigured))
else:
@@ -928,7 +944,7 @@ class BaseFTPTestCase(unittest.TestCase):
self.factory = FTPFactory(portal=p)
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
self.portNum = self.port.getHost().port
- self.download_handler = FTPDownloadHandler(Settings())
+ self.download_handler = FTPDownloadHandler(get_crawler())
self.addCleanup(self.port.stopListening)
def tearDown(self):
@@ -1042,7 +1058,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
userAnonymous=self.username)
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
self.portNum = self.port.getHost().port
- self.download_handler = FTPDownloadHandler(Settings())
+ self.download_handler = FTPDownloadHandler(get_crawler())
self.addCleanup(self.port.stopListening)
def tearDown(self):
@@ -1052,7 +1068,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
class DataURITestCase(unittest.TestCase):
def setUp(self):
- self.download_handler = DataURIDownloadHandler(Settings())
+ self.download_handler = DataURIDownloadHandler()
self.download_request = self.download_handler.download_request
self.spider = Spider('foo')