Scrapy, a fast high-level web crawling & scraping framework for Python.
using botocore for s3 request signing: proof of concept
Konstantin Lopuhineaf3a239e48a
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 38cfd1e10..d3feb9815 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,7 +1,9 @@+import sixfrom six.moves.urllib.parse import unquotefrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.python import to_unicodefrom .http import HTTPDownloadHandler@@ -37,10 +39,6 @@ class S3DownloadHandler(object):def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \httpdownloadhandler=HTTPDownloadHandler, **kw):- _S3Connection = get_s3_connection()- if _S3Connection is None:- raise NotConfigured("missing boto library")-if not aws_access_key_id:aws_access_key_id = settings['AWS_ACCESS_KEY_ID']if not aws_secret_access_key:@@ -53,10 +51,27 @@ class S3DownloadHandler(object):if anon is None and not aws_access_key_id and not aws_secret_access_key:kw['anon'] = True+ self._signer = Nonetry:- self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)- except Exception as ex:- raise NotConfigured(str(ex))+ import botocore.auth+ import botocore.credentials+ except ImportError:+ if six.PY3:+ raise NotConfigured("missing botocore library")+ _S3Connection = get_s3_connection()+ if _S3Connection is None:+ raise NotConfigured("missing botocore or boto library")+ try:+ self.conn = _S3Connection(+ aws_access_key_id, aws_secret_access_key, **kw)+ except Exception as ex:+ raise NotConfigured(str(ex))+ else:+ SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']+ # TODO - anon+ self._signer = SignerCls(botocore.credentials.Credentials(+ aws_access_key_id, aws_secret_access_key))+self._download_http = httpdownloadhandler(settings).download_requestdef download_request(self, request, spider):@@ -65,12 +80,28 @@ class S3DownloadHandler(object):bucket = p.hostnamepath = p.path + '?' + p.query if p.query else p.pathurl = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)- signed_headers = self.conn.make_request(+ if self._signer is not None:+ import botocore.awsrequest+ from botocore.vendored.requests.structures import CaseInsensitiveDict+ print(url, request.headers)+ awsrequest = botocore.awsrequest.AWSRequest(method=request.method,- bucket=bucket,- key=unquote(p.path),- query_args=unquote(p.query),- headers=request.headers,+ url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),+ # TODO - move to a header method+ headers=CaseInsensitiveDict(+ (to_unicode(key), to_unicode(b','.join(value)))+ for key, value in request.headers.items()),data=request.body)- httpreq = request.replace(url=url, headers=signed_headers)- return self._download_http(httpreq, spider)+ self._signer.add_auth(awsrequest)+ request = request.replace(+ url=url, headers=awsrequest.headers.items())+ else:+ signed_headers = self.conn.make_request(+ method=request.method,+ bucket=bucket,+ key=unquote(p.path),+ query_args=unquote(p.query),+ headers=request.headers,+ data=request.body)+ request = request.replace(url=url, headers=signed_headers)+ return self._download_http(request, spider)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 56608bfc6..06e232503 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -432,13 +432,20 @@ class HttpDownloadHandlerMock(object):return request-class S3AnonTestCase(unittest.TestCase):- try:- import boto- except ImportError:- skip = 'missing boto library'+class BaseS3TestCase(unittest.TestCase):if six.PY3:- skip = 'S3 not supported on Py3'+ try:+ import botocore+ except ImportError:+ skip = 'missing botocore library'+ else:+ try:+ import boto+ except ImportError:+ skip = 'missing boto library'+++class S3AnonTestCase(BaseS3TestCase):def setUp(self):self.s3reqh = S3DownloadHandler(Settings(),@@ -457,12 +464,6 @@ class S3AnonTestCase(unittest.TestCase):class S3TestCase(unittest.TestCase):download_handler_cls = S3DownloadHandler- try:- import boto- except ImportError:- skip = 'missing boto library'- if six.PY3:- skip = 'S3 not supported on Py3'# test use same example keys than amazon developer guide# http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf@@ -484,7 +485,7 @@ class S3TestCase(unittest.TestCase):headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')def test_request_signing2(self):# puts an object into the johnsmith bucket.@@ -495,7 +496,7 @@ class S3TestCase(unittest.TestCase):})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')def test_request_signing3(self):# lists the content of the johnsmith bucket.@@ -506,7 +507,7 @@ class S3TestCase(unittest.TestCase):})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')def test_request_signing4(self):# fetches the access control policy sub-resource for the 'johnsmith' bucket.@@ -514,7 +515,7 @@ class S3TestCase(unittest.TestCase):method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')def test_request_signing5(self):# deletes an object from the 'johnsmith' bucket using the@@ -526,7 +527,7 @@ class S3TestCase(unittest.TestCase):})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')def test_request_signing6(self):# uploads an object to a CNAME style virtual hosted bucket with metadata.@@ -547,7 +548,7 @@ class S3TestCase(unittest.TestCase):})httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'], \- 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')def test_request_signing7(self):# ensure that spaces are quoted properly before signing@@ -561,7 +562,7 @@ class S3TestCase(unittest.TestCase):httpreq = self.download_request(req, self.spider)self.assertEqual(httpreq.headers['Authorization'],- 'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')+ b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')class FTPTestCase(unittest.TestCase):