Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

expose AWS_REGION_NAME in S3FeedStorage

Kevin Lloyd Bernal9a1bf40c2f7a
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2218f45e7..700775e4b 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -210,10 +210,12 @@ passed through the following settings:
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
-You can also define a custom ACL and custom endpoint for exported feeds using this setting:
+You can also define a custom ACL, custom endpoint, and region name for exported
+feeds using these settings:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
+- :setting:`AWS_REGION_NAME`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c81f396cb..84724640b 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage):
secret_key=None,
acl=None,
endpoint_url=None,
+ region_name=None,
*,
feed_options=None,
session_token=None,
@@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
+ self.region_name = region_name
if IS_BOTO3_AVAILABLE:
import boto3.session
@@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage):
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
+ region_name=self.region_name,
)
else:
warnings.warn(
@@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage):
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
+ region_name=self.region_name,
)
if feed_options and feed_options.get("overwrite", True) is False:
@@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage):
session_token=crawler.settings["AWS_SESSION_TOKEN"],
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
+ region_name=crawler.settings["AWS_REGION_NAME"] or None,
feed_options=feed_options,
)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 96bb688ab..9e4fb53bf 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, "secret_key")
self.assertEqual(storage.endpoint_url, "https://example.com")
+ def test_init_with_region_name(self):
+ region_name = "ap-east-1"
+ storage = S3FeedStorage(
+ "s3://mybucket/export.csv",
+ "access_key",
+ "secret_key",
+ region_name=region_name,
+ )
+ self.assertEqual(storage.access_key, "access_key")
+ self.assertEqual(storage.secret_key, "secret_key")
+ self.assertEqual(storage.region_name, region_name)
+ self.assertEqual(storage.s3_client._client_config.region_name, region_name)
+
def test_from_crawler_without_acl(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
@@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, "secret_key")
self.assertEqual(storage.endpoint_url, None)
+ def test_without_region_name(self):
+ settings = {
+ "AWS_ACCESS_KEY_ID": "access_key",
+ "AWS_SECRET_ACCESS_KEY": "secret_key",
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(
+ crawler,
+ "s3://mybucket/export.csv",
+ )
+ self.assertEqual(storage.access_key, "access_key")
+ self.assertEqual(storage.secret_key, "secret_key")
+ self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1")
+
def test_from_crawler_with_acl(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
@@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.secret_key, "secret_key")
self.assertEqual(storage.endpoint_url, "https://example.com")
+ def test_from_crawler_with_region_name(self):
+ region_name = "ap-east-1"
+ settings = {
+ "AWS_ACCESS_KEY_ID": "access_key",
+ "AWS_SECRET_ACCESS_KEY": "secret_key",
+ "AWS_REGION_NAME": region_name,
+ }
+ crawler = get_crawler(settings_dict=settings)
+ storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
+ self.assertEqual(storage.access_key, "access_key")
+ self.assertEqual(storage.secret_key, "secret_key")
+ self.assertEqual(storage.region_name, region_name)
+ self.assertEqual(storage.s3_client._client_config.region_name, region_name)
+
@defer.inlineCallbacks
def test_store_without_acl(self):
storage = S3FeedStorage(