Scrapy, a fast high-level web crawling & scraping framework for Python.
expose AWS_REGION_NAME in S3FeedStorage
Kevin Lloyd Bernal9a1bf40c2f7a
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 2218f45e7..700775e4b 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -210,10 +210,12 @@ passed through the following settings:.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys-You can also define a custom ACL and custom endpoint for exported feeds using this setting:+You can also define a custom ACL, custom endpoint, and region name for exported+feeds using these settings:- :setting:`FEED_STORAGE_S3_ACL`- :setting:`AWS_ENDPOINT_URL`+- :setting:`AWS_REGION_NAME`The default value for the ``overwrite`` key in the :setting:`FEEDS` for thisstorage backend is: ``True``.diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c81f396cb..84724640b 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -169,6 +169,7 @@ class S3FeedStorage(BlockingFeedStorage):secret_key=None,acl=None,endpoint_url=None,+ region_name=None,*,feed_options=None,session_token=None,@@ -183,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):self.keyname = u.path[1:] # remove first "/"self.acl = aclself.endpoint_url = endpoint_url+ self.region_name = region_nameif IS_BOTO3_AVAILABLE:import boto3.session@@ -195,6 +197,7 @@ class S3FeedStorage(BlockingFeedStorage):aws_secret_access_key=self.secret_key,aws_session_token=self.session_token,endpoint_url=self.endpoint_url,+ region_name=self.region_name,)else:warnings.warn(@@ -213,6 +216,7 @@ class S3FeedStorage(BlockingFeedStorage):aws_secret_access_key=self.secret_key,aws_session_token=self.session_token,endpoint_url=self.endpoint_url,+ region_name=self.region_name,)if feed_options and feed_options.get("overwrite", True) is False:@@ -232,6 +236,7 @@ class S3FeedStorage(BlockingFeedStorage):session_token=crawler.settings["AWS_SESSION_TOKEN"],acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,+ region_name=crawler.settings["AWS_REGION_NAME"] or None,feed_options=feed_options,)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 96bb688ab..9e4fb53bf 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -349,6 +349,19 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, "secret_key")self.assertEqual(storage.endpoint_url, "https://example.com")+ def test_init_with_region_name(self):+ region_name = "ap-east-1"+ storage = S3FeedStorage(+ "s3://mybucket/export.csv",+ "access_key",+ "secret_key",+ region_name=region_name,+ )+ self.assertEqual(storage.access_key, "access_key")+ self.assertEqual(storage.secret_key, "secret_key")+ self.assertEqual(storage.region_name, region_name)+ self.assertEqual(storage.s3_client._client_config.region_name, region_name)+def test_from_crawler_without_acl(self):settings = {"AWS_ACCESS_KEY_ID": "access_key",@@ -377,6 +390,20 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, "secret_key")self.assertEqual(storage.endpoint_url, None)+ def test_without_region_name(self):+ settings = {+ "AWS_ACCESS_KEY_ID": "access_key",+ "AWS_SECRET_ACCESS_KEY": "secret_key",+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(+ crawler,+ "s3://mybucket/export.csv",+ )+ self.assertEqual(storage.access_key, "access_key")+ self.assertEqual(storage.secret_key, "secret_key")+ self.assertEqual(storage.s3_client._client_config.region_name, "us-east-1")+def test_from_crawler_with_acl(self):settings = {"AWS_ACCESS_KEY_ID": "access_key",@@ -404,6 +431,20 @@ class S3FeedStorageTest(unittest.TestCase):self.assertEqual(storage.secret_key, "secret_key")self.assertEqual(storage.endpoint_url, "https://example.com")+ def test_from_crawler_with_region_name(self):+ region_name = "ap-east-1"+ settings = {+ "AWS_ACCESS_KEY_ID": "access_key",+ "AWS_SECRET_ACCESS_KEY": "secret_key",+ "AWS_REGION_NAME": region_name,+ }+ crawler = get_crawler(settings_dict=settings)+ storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")+ self.assertEqual(storage.access_key, "access_key")+ self.assertEqual(storage.secret_key, "secret_key")+ self.assertEqual(storage.region_name, region_name)+ self.assertEqual(storage.s3_client._client_config.region_name, region_name)+@defer.inlineCallbacksdef test_store_without_acl(self):storage = S3FeedStorage(