Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Pass args/kwargs in S3DownloadHandler.from_crawler, update tests

Eugenio Lacuestae43f37fff3cb
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 220296fb3..99a3a7925 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -69,8 +69,8 @@ class S3DownloadHandler(object):
self._download_http = httpdownloadhandler(crawler).download_request
@classmethod
- def from_crawler(cls, crawler):
- return cls(crawler)
+ def from_crawler(cls, crawler, *args, **kwargs):
+ return cls(crawler, *args, **kwargs)
def download_request(self, request, spider):
p = urlparse_cached(request)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 815342219..eff5653f0 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -250,7 +250,7 @@ class HttpTestCase(unittest.TestCase):
else:
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
self.portno = self.port.getHost().port
- self.download_handler = self.download_handler_cls(get_crawler())
+ self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@@ -490,7 +490,7 @@ class Http11TestCase(HttpTestCase):
return self.test_download_broken_content_allow_data_loss('broken-chunked')
def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
- download_handler = self.download_handler_cls(
+ download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
)
request = Request(self.getURL(url))
@@ -510,7 +510,7 @@ class Https11TestCase(Http11TestCase):
@defer.inlineCallbacks
def test_tls_logging(self):
- download_handler = self.download_handler_cls(
+ download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
)
try:
@@ -579,7 +579,7 @@ class Https11CustomCiphers(unittest.TestCase):
0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
interface=self.host)
self.portno = self.port.getHost().port
- self.download_handler = self.download_handler_cls(
+ self.download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
)
self.download_request = self.download_handler.download_request
@@ -677,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
- self.download_handler = self.download_handler_cls(get_crawler())
+ self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@@ -750,7 +750,7 @@ class S3AnonTestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
- self.s3reqh = S3DownloadHandler(
+ self.s3reqh = S3DownloadHandler.from_crawler(
crawler=get_crawler(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
@@ -779,10 +779,10 @@ class S3TestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
- s3reqh = S3DownloadHandler(
- get_crawler(),
- self.AWS_ACCESS_KEY_ID,
- self.AWS_SECRET_ACCESS_KEY,
+ s3reqh = S3DownloadHandler.from_crawler(
+ crawler=get_crawler(),
+ aws_access_key_id=self.AWS_ACCESS_KEY_ID,
+ aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock,
)
self.download_request = s3reqh.download_request
@@ -804,7 +804,7 @@ class S3TestCase(unittest.TestCase):
def test_extra_kw(self):
try:
- S3DownloadHandler(get_crawler(), extra_kw=True)
+ S3DownloadHandler.from_crawler(get_crawler(), extra_kw=True)
except Exception as e:
self.assertIsInstance(e, (TypeError, NotConfigured))
else: