Scrapy, a fast high-level web crawling & scraping framework for Python.
refactoring
Victor Torresad83ffdf1f4d
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex eb0802261..ca30322be 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -93,7 +93,7 @@ class FileFeedStorage(object):class S3FeedStorage(BlockingFeedStorage):- def __init__(self, uri, access_key=None, secret_key=None):+ def __init__(self, uri, access_key=None, secret_key=None, acl=None):# BEGIN Backwards compatibility for initialising without keys (and# without using from_crawler)no_defaults = access_key is None and secret_key is None@@ -118,7 +118,7 @@ class S3FeedStorage(BlockingFeedStorage):self.secret_key = u.password or secret_keyself.is_botocore = is_botocore()self.keyname = u.path[1:] # remove first "/"- self.policy = settings.get('FEED_STORAGE_S3_ACL', 'private')+ self.acl = aclif self.is_botocore:import botocore.sessionsession = botocore.session.get_session()@@ -132,19 +132,28 @@ class S3FeedStorage(BlockingFeedStorage):@classmethoddef from_crawler(cls, crawler, uri):return cls(uri, crawler.settings['AWS_ACCESS_KEY_ID'],- crawler.settings['AWS_SECRET_ACCESS_KEY'])+ crawler.settings['AWS_SECRET_ACCESS_KEY'],+ crawler.settings.get('FEED_STORAGE_S3_ACL'))def _store_in_thread(self, file):file.seek(0)if self.is_botocore:+ kwargs = dict()+ if self.acl:+ kwargs.update(dict(ACL=self.acl))+self.s3_client.put_object(Bucket=self.bucketname, Key=self.keyname, Body=file,- ACL=self.policy)+ **kwargs)else:conn = self.connect_s3(self.access_key, self.secret_key)bucket = conn.get_bucket(self.bucketname, validate=False)key = bucket.new_key(self.keyname)- key.set_contents_from_file(file, policy=self.policy)+ kwargs = dict()+ if self.acl:+ kwargs.update(dict(policy=self.acl))++ key.set_contents_from_file(file, **kwargs)key.close()diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex e46c8c14e..b07635cb0 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -18,6 +18,7 @@ from tests import mockfrom tests.mockserver import MockServerfrom w3lib.url import path_to_file_uri+import botocore.clientimport scrapyfrom scrapy.exporters import CsvItemExporterfrom scrapy.extensions.feedexport import (@@ -186,6 +187,89 @@ class S3FeedStorageTest(unittest.TestCase):content = get_s3_content_and_delete(u.hostname, u.path[1:])self.assertEqual(content, expected_content)+ def test_init_without_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ def test_init_with_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ def test_from_crawler_without_acl(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ def test_from_crawler_with_acl(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ 'FEED_STORAGE_S3_ACL': 'custom-acl',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ def test_store_in_thread_without_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:+ storage._store_in_thread(BytesIO(b'test file'))+ operation_name, api_params = _make_api_call_mock.call_args[0]+ self.assertEqual(operation_name, 'PutObject')+ self.assertNotIn('ACL', api_params)++ def test_store_in_thread_with_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ with mock.patch('botocore.client.BaseClient._make_api_call') as _make_api_call_mock:+ storage._store_in_thread(BytesIO(b'test file'))+ operation_name, api_params = _make_api_call_mock.call_args[0]+ self.assertEqual(operation_name, 'PutObject')+ self.assertEqual(api_params.get('ACL'), 'custom-acl')+class StdoutFeedStorageTest(unittest.TestCase):