Scrapy, a fast high-level web crawling & scraping framework for Python.
mock date in s3 tests when using botocore
Konstantin Lopuhin7748ee6bba8e
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 0903b84ad..cb2bb46b1 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -68,10 +68,10 @@ class S3DownloadHandler(object):except Exception as ex:raise NotConfigured(str(ex))else:- SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']- # TODO - anon- self._signer = SignerCls(botocore.credentials.Credentials(- aws_access_key_id, aws_secret_access_key))+ if not self.anon:+ SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']+ self._signer = SignerCls(botocore.credentials.Credentials(+ aws_access_key_id, aws_secret_access_key))self._download_http = httpdownloadhandler(settings).download_requestdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 0f79a208d..6c4d2e0db 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -1,5 +1,10 @@import osimport six+import contextlib+try:+ from unittest import mock+except ImportError:+ import mockfrom twisted.trial import unittestfrom twisted.protocols.policies import WrappingFactory@@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object):class BaseS3TestCase(unittest.TestCase):- if six.PY3:- try:- import botocore- except ImportError:+ try:+ import botocore+ except ImportError:+ if six.PY2:+ try:+ import boto+ except ImportError:+ skip = 'missing botocore or boto library'+ else:skip = 'missing botocore library'- else:- try:- import boto- except ImportError:- skip = 'missing boto library'class S3AnonTestCase(BaseS3TestCase):@@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase):httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')-class S3TestCase(unittest.TestCase):+class S3TestCase(BaseS3TestCase):download_handler_cls = S3DownloadHandler# test use same example keys than amazon developer guide@@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase):self.download_request = s3reqh.download_requestself.spider = Spider('foo')+ @contextlib.contextmanager+ def _mocked_date(self, date):+ try:+ import botocore.auth+ except ImportError:+ yield+ else:+ # We need to mock botocore.auth.formatdate, because otherwise+ # botocore overrides Date header with current date and time+ # and Authorization header is different each time+ with mock.patch('botocore.auth.formatdate') as mock_formatdate:+ mock_formatdate.return_value = date+ yield+def test_request_signing1(self):# gets an object from the johnsmith bucket.- req = Request('s3://johnsmith/photos/puppy.jpg',- headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})- httpreq = self.download_request(req, self.spider)+ date ='Tue, 27 Mar 2007 19:36:42 +0000'+ req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')def test_request_signing2(self):# puts an object into the johnsmith bucket.+ date = 'Tue, 27 Mar 2007 21:15:45 +0000'req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={'Content-Type': 'image/jpeg',- 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',+ 'Date': date,'Content-Length': '94328',})- httpreq = self.download_request(req, self.spider)+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')def test_request_signing3(self):# lists the content of the johnsmith bucket.+ date = 'Tue, 27 Mar 2007 19:42:41 +0000'req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \method='GET', headers={'User-Agent': 'Mozilla/5.0',- 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',+ 'Date': date,})- httpreq = self.download_request(req, self.spider)+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')def test_request_signing4(self):# fetches the access control policy sub-resource for the 'johnsmith' bucket.- req = Request('s3://johnsmith/?acl', \- method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})- httpreq = self.download_request(req, self.spider)+ date = 'Tue, 27 Mar 2007 19:44:46 +0000'+ req = Request('s3://johnsmith/?acl',+ method='GET', headers={'Date': date})+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')def test_request_signing5(self):# deletes an object from the 'johnsmith' bucket using the# path-style and Date alternative.+ date = 'Tue, 27 Mar 2007 21:20:27 +0000'req = Request('s3://johnsmith/photos/puppy.jpg', \method='DELETE', headers={- 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',+ 'Date': date,'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',})- httpreq = self.download_request(req, self.spider)- self.assertEqual(httpreq.headers['Authorization'], \- b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)+ # botocore does not override Date with x-amz-date+ self.assertIn(httpreq.headers['Authorization'], [+ b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',+ b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])def test_request_signing6(self):# uploads an object to a CNAME style virtual hosted bucket with metadata.+ date = 'Tue, 27 Mar 2007 21:06:08 +0000'req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \method='PUT', headers={'User-Agent': 'curl/7.15.5','Host': 'static.johnsmith.net:8080',- 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',+ 'Date': date,'x-amz-acl': 'public-read','content-type': 'application/x-download','Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',@@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase):'Content-Encoding': 'gzip','Content-Length': '5913339',})- httpreq = self.download_request(req, self.spider)+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')def test_request_signing7(self):# ensure that spaces are quoted properly before signing+ date = 'Tue, 27 Mar 2007 19:42:41 +0000'req = Request(("s3://johnsmith/photos/my puppy.jpg""?response-content-disposition=my puppy.jpg"),method='GET',- headers={- 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',- })- httpreq = self.download_request(req, self.spider)+ headers={'Date': date},+ )+ with self._mocked_date(date):+ httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'],b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')