Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Adding an option to set ACL while uploading the blob to GCS

Júlio César Batista2bbbd02bda36
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 8347b42ca..fbbf9bb97 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -148,15 +148,20 @@ class S3FeedStorage(BlockingFeedStorage):
class GCSFeedStorage(BlockingFeedStorage):
- def __init__(self, uri, project_id):
+ def __init__(self, uri, project_id, acl):
self.project_id = project_id
+ self.acl = acl
u = urlparse(uri)
self.bucket_name = u.hostname
self.blob_name = u.path[1:] # remove first "/"
@classmethod
def from_crawler(cls, crawler, uri):
- return cls(uri, crawler.settings['GCS_PROJECT_ID'])
+ return cls(
+ uri,
+ crawler.settings['GCS_PROJECT_ID'],
+ crawler.settings['FEED_STORAGE_GCS_ACL']
+ )
def _store_in_thread(self, file):
file.seek(0)
@@ -164,7 +169,7 @@ class GCSFeedStorage(BlockingFeedStorage):
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
blob = bucket.blob(self.blob_name)
- blob.upload_from_file(file)
+ blob.upload_from_file(file, predefined_acl=self.acl)
class FTPFeedStorage(BlockingFeedStorage):
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8769c01ba..5d2862980 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -159,6 +159,8 @@ FEED_EXPORTERS_BASE = {
}
FEED_EXPORT_INDENT = 0
+FEED_STORAGE_GCS_ACL = None
+
FILES_STORE_S3_ACL = 'private'
FILES_STORE_GCS_ACL = ''
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 6d23c68c3..5cbca6d28 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -191,17 +191,18 @@ class S3FeedStorageTest(unittest.TestCase):
class GCSFeedStorageTest(unittest.TestCase):
@mock.patch('scrapy.conf.settings',
- new={'GCS_PROJECT_ID': 'conf_id' }, create=True)
+ new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
def test_parse_settings(self):
try:
from google.cloud.storage import Client
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
- settings = {'GCS_PROJECT_ID': '123' }
+ settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, 'gcs://mybucket/export.csv')
assert storage.project_id == '123'
+ assert storage.acl == 'publicRead'
assert storage.bucket_name == 'mybucket'
assert storage.blob_name == 'export.csv'
@@ -214,19 +215,20 @@ class GCSFeedStorageTest(unittest.TestCase):
uri = 'gcs://mybucket/export.csv'
project_id = 'myproject-123'
+ acl = 'publicRead'
(client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
with mock.patch('google.cloud.storage.Client') as m:
m.return_value = client_mock
f = mock.Mock()
- storage = GCSFeedStorage(uri, project_id)
+ storage = GCSFeedStorage(uri, project_id, acl)
yield storage.store(f)
f.seek.assert_called_once_with(0)
m.assert_called_once_with(project=project_id)
client_mock.get_bucket.assert_called_once_with('mybucket')
bucket_mock.blob.assert_called_once_with('export.csv')
- blob_mock.upload_from_file.assert_called_once_with(f)
+ blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
class StdoutFeedStorageTest(unittest.TestCase):