Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into feat-685

Adrián Chavesba13de29e101
diff --cc docs/topics/feed-exports.rst
index 0957a5997,cf70b8aca..9f837221e
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@@ -184,28 -184,11 +184,32 @@@ passed through the following settings
* :setting:`AWS_ACCESS_KEY_ID`
* :setting:`AWS_SECRET_ACCESS_KEY`
-
++
+ You can also define a custom ACL for exported feeds using this setting:
+
+ * :setting:`FEED_STORAGE_S3_ACL`
+.. _topics-feed-storage-gcs:
+
+Google Cloud Storage (GCS)
+--------------------------
+
+The feeds are stored on `Google Cloud Storage`_.
+
+ * URI scheme: ``gs``
+ * Example URIs:
+
+ * ``gs://mybucket/path/to/export.csv``
+
+ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.
+
+For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
+
+You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
+
+ * :setting:`FEED_STORAGE_GCS_ACL`
+ * :setting:`GCS_PROJECT_ID`
+
.. _topics-feed-storage-stdout:
Standard output
@@@ -387,4 -382,4 +403,5 @@@ format in :setting:`FEED_EXPORTERS`. E.
.. _Amazon S3: https://aws.amazon.com/s3/
.. _boto: https://github.com/boto/boto
.. _botocore: https://github.com/boto/botocore
- .. _Google Cloud Storage: https://cloud.google.com/storage/
+ .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
++.. _Google Cloud Storage: https://cloud.google.com/storage/
diff --cc scrapy/settings/default_settings.py
index 50fcd1d0a,a800d39ab..a472f3786
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@@ -159,7 -158,7 +159,8 @@@ FEED_EXPORTERS_BASE =
}
FEED_EXPORT_INDENT = 0
+FEED_STORAGE_GCS_ACL = None
+ FEED_STORAGE_S3_ACL = ''
FILES_STORE_S3_ACL = 'private'
FILES_STORE_GCS_ACL = ''
diff --cc tests/test_feedexport.py
index 41df7d7af,eef0384cf..70a0bef7f
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -187,49 -186,152 +187,194 @@@ class S3FeedStorageTest(unittest.TestCa
content = get_s3_content_and_delete(u.hostname, u.path[1:])
self.assertEqual(content, expected_content)
+ def test_init_without_acl(self):
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, None)
+
+ def test_init_with_acl(self):
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ 'custom-acl'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, 'custom-acl')
+
+ def test_from_crawler_without_acl(self):
+ settings = {
+ 'AWS_ACCESS_KEY_ID': 'access_key',
+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ 's3://mybucket/export.csv'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, None)
+
+ def test_from_crawler_with_acl(self):
+ settings = {
+ 'AWS_ACCESS_KEY_ID': 'access_key',
+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',
+ 'FEED_STORAGE_S3_ACL': 'custom-acl',
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ 's3://mybucket/export.csv'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, 'custom-acl')
+
+ @defer.inlineCallbacks
+ def test_store_botocore_without_acl(self):
+ try:
+ import botocore
+ except ImportError:
+ raise unittest.SkipTest('botocore is required')
+
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, None)
+
+ storage.s3_client = mock.MagicMock()
+ yield storage.store(BytesIO(b'test file'))
+ self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1])
+
+ @defer.inlineCallbacks
+ def test_store_botocore_with_acl(self):
+ try:
+ import botocore
+ except ImportError:
+ raise unittest.SkipTest('botocore is required')
+
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ 'custom-acl'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, 'custom-acl')
+
+ storage.s3_client = mock.MagicMock()
+ yield storage.store(BytesIO(b'test file'))
+ self.assertEqual(
+ storage.s3_client.put_object.call_args[1].get('ACL'),
+ 'custom-acl'
+ )
+
+ @defer.inlineCallbacks
+ def test_store_not_botocore_without_acl(self):
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, None)
+
+ storage.is_botocore = False
+ storage.connect_s3 = mock.MagicMock()
+ self.assertFalse(storage.is_botocore)
+
+ yield storage.store(BytesIO(b'test file'))
+
+ conn = storage.connect_s3(*storage.connect_s3.call_args)
+ bucket = conn.get_bucket(*conn.get_bucket.call_args)
+ key = bucket.new_key(*bucket.new_key.call_args)
+ self.assertNotIn(
+ dict(policy='custom-acl'),
+ key.set_contents_from_file.call_args
+ )
+
+ @defer.inlineCallbacks
+ def test_store_not_botocore_with_acl(self):
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ 'custom-acl'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.acl, 'custom-acl')
+
+ storage.is_botocore = False
+ storage.connect_s3 = mock.MagicMock()
+ self.assertFalse(storage.is_botocore)
+
+ yield storage.store(BytesIO(b'test file'))
+
+ conn = storage.connect_s3(*storage.connect_s3.call_args)
+ bucket = conn.get_bucket(*conn.get_bucket.call_args)
+ key = bucket.new_key(*bucket.new_key.call_args)
+ self.assertIn(
+ dict(policy='custom-acl'),
+ key.set_contents_from_file.call_args
+ )
+
+class GCSFeedStorageTest(unittest.TestCase):
+
+ @mock.patch('scrapy.conf.settings',
+ new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
+ def test_parse_settings(self):
+ try:
+ from google.cloud.storage import Client
+ except ImportError:
+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+ settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
+ crawler = get_crawler(settings_dict=settings)
+ storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
+ assert storage.project_id == '123'
+ assert storage.acl == 'publicRead'
+ assert storage.bucket_name == 'mybucket'
+ assert storage.blob_name == 'export.csv'
+
+ @defer.inlineCallbacks
+ def test_store(self):
+ try:
+ from google.cloud.storage import Client
+ except ImportError:
+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
+
+ uri = 'gcs://mybucket/export.csv'
+ project_id = 'myproject-123'
+ acl = 'publicRead'
+ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+ with mock.patch('google.cloud.storage.Client') as m:
+ m.return_value = client_mock
+
+ f = mock.Mock()
+ storage = GCSFeedStorage(uri, project_id, acl)
+ yield storage.store(f)
+
+ f.seek.assert_called_once_with(0)
+ m.assert_called_once_with(project=project_id)
+ client_mock.get_bucket.assert_called_once_with('mybucket')
+ bucket_mock.blob.assert_called_once_with('export.csv')
+ blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+
class StdoutFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks