Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into feat-685
Adrián Chavesba13de29e101
diff --cc docs/topics/feed-exports.rstindex 0957a5997,cf70b8aca..9f837221e--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@@ -184,28 -184,11 +184,32 @@@ passed through the following settings* :setting:`AWS_ACCESS_KEY_ID`* :setting:`AWS_SECRET_ACCESS_KEY`-+++ You can also define a custom ACL for exported feeds using this setting:++ * :setting:`FEED_STORAGE_S3_ACL`+.. _topics-feed-storage-gcs:++Google Cloud Storage (GCS)+--------------------------++The feeds are stored on `Google Cloud Storage`_.++ * URI scheme: ``gs``+ * Example URIs:++ * ``gs://mybucket/path/to/export.csv``++ * Required external libraries: `google-cloud-storage <https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python>`_.++For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.++You can set a *Project ID* and *Access Control List (ACL)* through the following settings:++ * :setting:`FEED_STORAGE_GCS_ACL`+ * :setting:`GCS_PROJECT_ID`+.. _topics-feed-storage-stdout:Standard output@@@ -387,4 -382,4 +403,5 @@@ format in :setting:`FEED_EXPORTERS`. E... _Amazon S3: https://aws.amazon.com/s3/.. _boto: https://github.com/boto/boto.. _botocore: https://github.com/boto/botocore- .. _Google Cloud Storage: https://cloud.google.com/storage/+ .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl++.. _Google Cloud Storage: https://cloud.google.com/storage/diff --cc scrapy/settings/default_settings.pyindex 50fcd1d0a,a800d39ab..a472f3786--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@@ -159,7 -158,7 +159,8 @@@ FEED_EXPORTERS_BASE =}FEED_EXPORT_INDENT = 0+FEED_STORAGE_GCS_ACL = None+ FEED_STORAGE_S3_ACL = ''FILES_STORE_S3_ACL = 'private'FILES_STORE_GCS_ACL = ''diff --cc tests/test_feedexport.pyindex 41df7d7af,eef0384cf..70a0bef7f--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -187,49 -186,152 +187,194 @@@ class S3FeedStorageTest(unittest.TestCacontent = get_s3_content_and_delete(u.hostname, u.path[1:])self.assertEqual(content, expected_content)+ def test_init_without_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ def test_init_with_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ def test_from_crawler_without_acl(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ def test_from_crawler_with_acl(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ 'FEED_STORAGE_S3_ACL': 'custom-acl',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ @defer.inlineCallbacks+ def test_store_botocore_without_acl(self):+ try:+ import botocore+ except ImportError:+ raise unittest.SkipTest('botocore is required')++ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ storage.s3_client = mock.MagicMock()+ yield storage.store(BytesIO(b'test file'))+ self.assertNotIn('ACL', storage.s3_client.put_object.call_args[1])++ @defer.inlineCallbacks+ def test_store_botocore_with_acl(self):+ try:+ import botocore+ except ImportError:+ raise unittest.SkipTest('botocore is required')++ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ storage.s3_client = mock.MagicMock()+ yield storage.store(BytesIO(b'test file'))+ self.assertEqual(+ storage.s3_client.put_object.call_args[1].get('ACL'),+ 'custom-acl'+ )++ @defer.inlineCallbacks+ def test_store_not_botocore_without_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, None)++ storage.is_botocore = False+ storage.connect_s3 = mock.MagicMock()+ self.assertFalse(storage.is_botocore)++ yield storage.store(BytesIO(b'test file'))++ conn = storage.connect_s3(*storage.connect_s3.call_args)+ bucket = conn.get_bucket(*conn.get_bucket.call_args)+ key = bucket.new_key(*bucket.new_key.call_args)+ self.assertNotIn(+ dict(policy='custom-acl'),+ key.set_contents_from_file.call_args+ )++ @defer.inlineCallbacks+ def test_store_not_botocore_with_acl(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ 'custom-acl'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.acl, 'custom-acl')++ storage.is_botocore = False+ storage.connect_s3 = mock.MagicMock()+ self.assertFalse(storage.is_botocore)++ yield storage.store(BytesIO(b'test file'))++ conn = storage.connect_s3(*storage.connect_s3.call_args)+ bucket = conn.get_bucket(*conn.get_bucket.call_args)+ key = bucket.new_key(*bucket.new_key.call_args)+ self.assertIn(+ dict(policy='custom-acl'),+ key.set_contents_from_file.call_args+ )++class GCSFeedStorageTest(unittest.TestCase):++ @mock.patch('scrapy.conf.settings',+ new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)+ def test_parse_settings(self):+ try:+ from google.cloud.storage import Client+ except ImportError:+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")++ settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }+ crawler = get_crawler(settings_dict=settings)+ storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')+ assert storage.project_id == '123'+ assert storage.acl == 'publicRead'+ assert storage.bucket_name == 'mybucket'+ assert storage.blob_name == 'export.csv'++ @defer.inlineCallbacks+ def test_store(self):+ try:+ from google.cloud.storage import Client+ except ImportError:+ raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")++ uri = 'gcs://mybucket/export.csv'+ project_id = 'myproject-123'+ acl = 'publicRead'+ (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()+ with mock.patch('google.cloud.storage.Client') as m:+ m.return_value = client_mock++ f = mock.Mock()+ storage = GCSFeedStorage(uri, project_id, acl)+ yield storage.store(f)++ f.seek.assert_called_once_with(0)+ m.assert_called_once_with(project=project_id)+ client_mock.get_bucket.assert_called_once_with('mybucket')+ bucket_mock.blob.assert_called_once_with('export.csv')+ blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)+class StdoutFeedStorageTest(unittest.TestCase):@defer.inlineCallbacks