Scrapy, a fast high-level web crawling & scraping framework for Python.
S3FeedStorage: allow custom endpoint (#4998)
Türkalp Burak KAYRANCIOĞLUfcc6becc586e
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 7a4b054e9..af7fce852 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -201,9 +201,10 @@ passed through the following settings:- :setting:`AWS_ACCESS_KEY_ID`- :setting:`AWS_SECRET_ACCESS_KEY`-You can also define a custom ACL for exported feeds using this setting:+You can also define a custom ACL and custom endpoint for exported feeds using this setting:- :setting:`FEED_STORAGE_S3_ACL`+- :setting:`AWS_ENDPOINT_URL`This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 89dca12f4..84a79e32d 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -150,7 +150,7 @@ class FileFeedStorage:class S3FeedStorage(BlockingFeedStorage):- def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,+ def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,feed_options=None):if not is_botocore_available():raise NotConfigured('missing botocore library')@@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage):self.secret_key = u.password or secret_keyself.keyname = u.path[1:] # remove first "/"self.acl = acl+ self.endpoint_url = endpoint_urlimport botocore.sessionsession = botocore.session.get_session()self.s3_client = session.create_client('s3', aws_access_key_id=self.access_key,- aws_secret_access_key=self.secret_key)+ aws_secret_access_key=self.secret_key,+ endpoint_url=self.endpoint_url)if feed_options and feed_options.get('overwrite', True) is False:logger.warning('S3 does not support appending to files. To ''suppress this warning, remove the overwrite '@@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage):access_key=crawler.settings['AWS_ACCESS_KEY_ID'],secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,+ endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,feed_options=feed_options,)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 81437b011..da0b2c786 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, 'secret_key')self.assertEqual(storage.acl, 'custom-acl')+ def test_init_with_endpoint_url(self):+ storage = S3FeedStorage(+ 's3://mybucket/export.csv',+ 'access_key',+ 'secret_key',+ endpoint_url='https://example.com'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.endpoint_url, 'https://example.com')+def test_from_crawler_without_acl(self):settings = {'AWS_ACCESS_KEY_ID': 'access_key',@@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, 'secret_key')self.assertEqual(storage.acl, None)+ def test_without_endpoint_url(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv',+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.endpoint_url, None)+def test_from_crawler_with_acl(self):settings = {'AWS_ACCESS_KEY_ID': 'access_key',@@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, 'secret_key')self.assertEqual(storage.acl, 'custom-acl')+ def test_from_crawler_with_endpoint_url(self):+ settings = {+ 'AWS_ACCESS_KEY_ID': 'access_key',+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',+ 'AWS_ENDPOINT_URL': 'https://example.com',+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ 's3://mybucket/export.csv'+ )+ self.assertEqual(storage.access_key, 'access_key')+ self.assertEqual(storage.secret_key, 'secret_key')+ self.assertEqual(storage.endpoint_url, 'https://example.com')+@defer.inlineCallbacksdef test_store_botocore_without_acl(self):skip_if_no_boto()@@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):class S3FeedStorageWithoutFeedOptions(S3FeedStorage):- def __init__(self, uri, access_key, secret_key, acl):- super().__init__(uri, access_key, secret_key, acl)+ def __init__(self, uri, access_key, secret_key, acl, endpoint_url):+ super().__init__(uri, access_key, secret_key, acl, endpoint_url)class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):