Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

mock date in s3 tests when using botocore

Konstantin Lopuhin7748ee6bba8e
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 0903b84ad..cb2bb46b1 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -68,10 +68,10 @@ class S3DownloadHandler(object):
except Exception as ex:
raise NotConfigured(str(ex))
else:
- SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
- # TODO - anon
- self._signer = SignerCls(botocore.credentials.Credentials(
- aws_access_key_id, aws_secret_access_key))
+ if not self.anon:
+ SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
+ self._signer = SignerCls(botocore.credentials.Credentials(
+ aws_access_key_id, aws_secret_access_key))
self._download_http = httpdownloadhandler(settings).download_request
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 0f79a208d..6c4d2e0db 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,5 +1,10 @@
import os
import six
+import contextlib
+try:
+ from unittest import mock
+except ImportError:
+ import mock
from twisted.trial import unittest
from twisted.protocols.policies import WrappingFactory
@@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object):
class BaseS3TestCase(unittest.TestCase):
- if six.PY3:
- try:
- import botocore
- except ImportError:
+ try:
+ import botocore
+ except ImportError:
+ if six.PY2:
+ try:
+ import boto
+ except ImportError:
+ skip = 'missing botocore or boto library'
+ else:
skip = 'missing botocore library'
- else:
- try:
- import boto
- except ImportError:
- skip = 'missing boto library'
class S3AnonTestCase(BaseS3TestCase):
@@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase):
httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
-class S3TestCase(unittest.TestCase):
+class S3TestCase(BaseS3TestCase):
download_handler_cls = S3DownloadHandler
# test use same example keys than amazon developer guide
@@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase):
self.download_request = s3reqh.download_request
self.spider = Spider('foo')
+ @contextlib.contextmanager
+ def _mocked_date(self, date):
+ try:
+ import botocore.auth
+ except ImportError:
+ yield
+ else:
+ # We need to mock botocore.auth.formatdate, because otherwise
+ # botocore overrides Date header with current date and time
+ # and Authorization header is different each time
+ with mock.patch('botocore.auth.formatdate') as mock_formatdate:
+ mock_formatdate.return_value = date
+ yield
+
def test_request_signing1(self):
# gets an object from the johnsmith bucket.
- req = Request('s3://johnsmith/photos/puppy.jpg',
- headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
- httpreq = self.download_request(req, self.spider)
+ date ='Tue, 27 Mar 2007 19:36:42 +0000'
+ req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
def test_request_signing2(self):
# puts an object into the johnsmith bucket.
+ date = 'Tue, 27 Mar 2007 21:15:45 +0000'
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
'Content-Type': 'image/jpeg',
- 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
+ 'Date': date,
'Content-Length': '94328',
})
- httpreq = self.download_request(req, self.spider)
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
def test_request_signing3(self):
# lists the content of the johnsmith bucket.
+ date = 'Tue, 27 Mar 2007 19:42:41 +0000'
req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
method='GET', headers={
'User-Agent': 'Mozilla/5.0',
- 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+ 'Date': date,
})
- httpreq = self.download_request(req, self.spider)
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
- req = Request('s3://johnsmith/?acl', \
- method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
- httpreq = self.download_request(req, self.spider)
+ date = 'Tue, 27 Mar 2007 19:44:46 +0000'
+ req = Request('s3://johnsmith/?acl',
+ method='GET', headers={'Date': date})
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
def test_request_signing5(self):
# deletes an object from the 'johnsmith' bucket using the
# path-style and Date alternative.
+ date = 'Tue, 27 Mar 2007 21:20:27 +0000'
req = Request('s3://johnsmith/photos/puppy.jpg', \
method='DELETE', headers={
- 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
+ 'Date': date,
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
})
- httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
+ # botocore does not override Date with x-amz-date
+ self.assertIn(httpreq.headers['Authorization'], [
+ b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',
+ b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.
+ date = 'Tue, 27 Mar 2007 21:06:08 +0000'
req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
method='PUT', headers={
'User-Agent': 'curl/7.15.5',
'Host': 'static.johnsmith.net:8080',
- 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
+ 'Date': date,
'x-amz-acl': 'public-read',
'content-type': 'application/x-download',
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
@@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase):
'Content-Encoding': 'gzip',
'Content-Length': '5913339',
})
- httpreq = self.download_request(req, self.spider)
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
def test_request_signing7(self):
# ensure that spaces are quoted properly before signing
+ date = 'Tue, 27 Mar 2007 19:42:41 +0000'
req = Request(
("s3://johnsmith/photos/my puppy.jpg"
"?response-content-disposition=my puppy.jpg"),
method='GET',
- headers={
- 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
- })
- httpreq = self.download_request(req, self.spider)
+ headers={'Date': date},
+ )
+ with self._mocked_date(date):
+ httpreq = self.download_request(req, self.spider)
self.assertEqual(
httpreq.headers['Authorization'],
b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')