Scrapy, a fast high-level web crawling & scraping framework for Python.
Adding an option to set ACL while uploading the blob to GCS
Júlio César Batista2bbbd02bda36
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 8347b42ca..fbbf9bb97 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage):class GCSFeedStorage(BlockingFeedStorage):- def __init__(self, uri, project_id):+ def __init__(self, uri, project_id, acl):self.project_id = project_id+ self.acl = aclu = urlparse(uri)self.bucket_name = u.hostnameself.blob_name = u.path[1:] # remove first "/"@classmethoddef from_crawler(cls, crawler, uri):- return cls(uri, crawler.settings['GCS_PROJECT_ID'])+ return cls(+ uri,+ crawler.settings['GCS_PROJECT_ID'],+ crawler.settings['FEED_STORAGE_GCS_ACL']+ )def _store_in_thread(self, file):file.seek(0)@@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage):client = Client(project=self.project_id)bucket = client.get_bucket(self.bucket_name)blob = bucket.blob(self.blob_name)- blob.upload_from_file(file)+ blob.upload_from_file(file, predefined_acl=self.acl)class FTPFeedStorage(BlockingFeedStorage):diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 8769c01ba..5d2862980 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = {}FEED_EXPORT_INDENT = 0+FEED_STORAGE_GCS_ACL = None+FILES_STORE_S3_ACL = 'private'FILES_STORE_GCS_ACL = ''diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 6d23c68c3..5cbca6d28 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase):class GCSFeedStorageTest(unittest.TestCase):@mock.patch('scrapy.conf.settings',- new={'GCS_PROJECT_ID': 'conf_id' }, create=True)+ new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)def test_parse_settings(self):try:from google.cloud.storage import Clientexcept ImportError:raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")- settings = {'GCS_PROJECT_ID': '123' }+ settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }crawler = get_crawler(settings_dict=settings)storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')assert storage.project_id == '123'+ assert storage.acl == 'publicRead'assert storage.bucket_name == 'mybucket'assert storage.blob_name == 'export.csv'@@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase):uri = 'gcs://mybucket/export.csv'project_id = 'myproject-123'+ acl = 'publicRead'(client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()with mock.patch('google.cloud.storage.Client') as m:m.return_value = client_mockf = mock.Mock()- storage = GCSFeedStorage(uri, project_id)+ storage = GCSFeedStorage(uri, project_id, acl)yield storage.store(f)f.seek.assert_called_once_with(0)m.assert_called_once_with(project=project_id)client_mock.get_bucket.assert_called_once_with('mybucket')bucket_mock.blob.assert_called_once_with('export.csv')- blob_mock.upload_from_file.assert_called_once_with(f)+ blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)class StdoutFeedStorageTest(unittest.TestCase):