Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

S3FeedStorage: allow custom endpoint (#4998)

Türkalp Burak KAYRANCIOĞLUfcc6becc586e
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 7a4b054e9..af7fce852 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -201,9 +201,10 @@ passed through the following settings:
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
-You can also define a custom ACL for exported feeds using this setting:
+You can also define a custom ACL and custom endpoint for exported feeds using this setting:
- :setting:`FEED_STORAGE_S3_ACL`
+- :setting:`AWS_ENDPOINT_URL`
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 89dca12f4..84a79e32d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -150,7 +150,7 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
- def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
+ def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
feed_options=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
@@ -160,11 +160,13 @@ class S3FeedStorage(BlockingFeedStorage):
self.secret_key = u.password or secret_key
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
+ self.endpoint_url = endpoint_url
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
- aws_secret_access_key=self.secret_key)
+ aws_secret_access_key=self.secret_key,
+ endpoint_url=self.endpoint_url)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
'suppress this warning, remove the overwrite '
@@ -178,6 +180,7 @@ class S3FeedStorage(BlockingFeedStorage):
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
+ endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
feed_options=feed_options,
)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 81437b011..da0b2c786 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -326,6 +326,17 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, 'secret_key')
self.assertEqual(storage.acl, 'custom-acl')
+ def test_init_with_endpoint_url(self):
+ storage = S3FeedStorage(
+ 's3://mybucket/export.csv',
+ 'access_key',
+ 'secret_key',
+ endpoint_url='https://example.com'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.endpoint_url, 'https://example.com')
+
def test_from_crawler_without_acl(self):
settings = {
'AWS_ACCESS_KEY_ID': 'access_key',
@@ -340,6 +351,20 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, 'secret_key')
self.assertEqual(storage.acl, None)
+ def test_without_endpoint_url(self):
+ settings = {
+ 'AWS_ACCESS_KEY_ID': 'access_key',
+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ 's3://mybucket/export.csv',
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.endpoint_url, None)
+
def test_from_crawler_with_acl(self):
settings = {
'AWS_ACCESS_KEY_ID': 'access_key',
@@ -355,6 +380,21 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, 'secret_key')
self.assertEqual(storage.acl, 'custom-acl')
+ def test_from_crawler_with_endpoint_url(self):
+ settings = {
+ 'AWS_ACCESS_KEY_ID': 'access_key',
+ 'AWS_SECRET_ACCESS_KEY': 'secret_key',
+ 'AWS_ENDPOINT_URL': 'https://example.com',
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ 's3://mybucket/export.csv'
+ )
+ self.assertEqual(storage.access_key, 'access_key')
+ self.assertEqual(storage.secret_key, 'secret_key')
+ self.assertEqual(storage.endpoint_url, 'https://example.com')
+
@defer.inlineCallbacks
def test_store_botocore_without_acl(self):
skip_if_no_boto()
@@ -1917,8 +1957,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
- def __init__(self, uri, access_key, secret_key, acl):
- super().__init__(uri, access_key, secret_key, acl)
+ def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
+ super().__init__(uri, access_key, secret_key, acl, endpoint_url)
class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):