Scrapy, a fast high-level web crawling & scraping framework for Python.
Pass args/kwargs in S3DownloadHandler.from_crawler, update tests
Eugenio Lacuestae43f37fff3cb
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 220296fb3..99a3a7925 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -69,8 +69,8 @@ class S3DownloadHandler(object):self._download_http = httpdownloadhandler(crawler).download_request@classmethod- def from_crawler(cls, crawler):- return cls(crawler)+ def from_crawler(cls, crawler, *args, **kwargs):+ return cls(crawler, *args, **kwargs)def download_request(self, request, spider):p = urlparse_cached(request)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 815342219..eff5653f0 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -250,7 +250,7 @@ class HttpTestCase(unittest.TestCase):else:self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)self.portno = self.port.getHost().port- self.download_handler = self.download_handler_cls(get_crawler())+ self.download_handler = self.download_handler_cls.from_crawler(get_crawler())self.download_request = self.download_handler.download_request@defer.inlineCallbacks@@ -490,7 +490,7 @@ class Http11TestCase(HttpTestCase):return self.test_download_broken_content_allow_data_loss('broken-chunked')def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):- download_handler = self.download_handler_cls(+ download_handler = self.download_handler_cls.from_crawler(get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False}))request = Request(self.getURL(url))@@ -510,7 +510,7 @@ class Https11TestCase(Http11TestCase):@defer.inlineCallbacksdef test_tls_logging(self):- download_handler = self.download_handler_cls(+ download_handler = self.download_handler_cls.from_crawler(get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True}))try:@@ -579,7 +579,7 @@ class Https11CustomCiphers(unittest.TestCase):0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),interface=self.host)self.portno = self.port.getHost().port- self.download_handler = self.download_handler_cls(+ self.download_handler = self.download_handler_cls.from_crawler(get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))self.download_request = self.download_handler.download_request@@ -677,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):wrapper = WrappingFactory(site)self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')self.portno = self.port.getHost().port- self.download_handler = self.download_handler_cls(get_crawler())+ self.download_handler = self.download_handler_cls.from_crawler(get_crawler())self.download_request = self.download_handler.download_request@defer.inlineCallbacks@@ -750,7 +750,7 @@ class S3AnonTestCase(unittest.TestCase):def setUp(self):skip_if_no_boto()- self.s3reqh = S3DownloadHandler(+ self.s3reqh = S3DownloadHandler.from_crawler(crawler=get_crawler(),httpdownloadhandler=HttpDownloadHandlerMock,#anon=True, # is implicit@@ -779,10 +779,10 @@ class S3TestCase(unittest.TestCase):def setUp(self):skip_if_no_boto()- s3reqh = S3DownloadHandler(- get_crawler(),- self.AWS_ACCESS_KEY_ID,- self.AWS_SECRET_ACCESS_KEY,+ s3reqh = S3DownloadHandler.from_crawler(+ crawler=get_crawler(),+ aws_access_key_id=self.AWS_ACCESS_KEY_ID,+ aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,httpdownloadhandler=HttpDownloadHandlerMock,)self.download_request = s3reqh.download_request@@ -804,7 +804,7 @@ class S3TestCase(unittest.TestCase):def test_extra_kw(self):try:- S3DownloadHandler(get_crawler(), extra_kw=True)+ S3DownloadHandler.from_crawler(get_crawler(), extra_kw=True)except Exception as e:self.assertIsInstance(e, (TypeError, NotConfigured))else: