Scrapy, a fast high-level web crawling & scraping framework for Python.
Add support for temporary security credential in AWS auth
laggardkernel7e4321f201a7
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 216a8bc52..af60de716 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -135,7 +135,7 @@ Here are some examples to illustrate:- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``-.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence+.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hencethey can also be used as storage URI parameters.@@ -200,6 +200,7 @@ passed through the following settings:- :setting:`AWS_ACCESS_KEY_ID`- :setting:`AWS_SECRET_ACCESS_KEY`+- :setting:`AWS_SESSION_TOKEN` (Optional)You can also define a custom ACL and custom endpoint for exported feeds using this setting:@@ -357,7 +358,7 @@ For instance::'item_export_kwargs': {'export_empty_fields': True,},- },+ },'/home/user/documents/items.xml': {'format': 'xml','fields': ['name', 'price'],diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 1290b4a5e..58daafa6f 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -204,6 +204,20 @@ Default: ``None``The AWS secret key used by code that requires access to `Amazon Web services`_,such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.+.. setting:: AWS_SESSION_TOKEN++AWS_SESSION_TOKEN+-----------------++Default: ``None`` (Optional)++The AWS security token used by code that requires access to `Amazon Web services`_,+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.++The security token is only required by a *temporary security credentials*.+Using of temporary security credentials is discouraged cause the credentials+are short term. It may expires before the scraping is done.+.. setting:: AWS_ENDPOINT_URLAWS_ENDPOINT_URLdiff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 31f1be31a..51ca1ed5e 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -10,6 +10,7 @@ class S3DownloadHandler:def __init__(self, settings, *,crawler=None,aws_access_key_id=None, aws_secret_access_key=None,+ aws_session_token=None,httpdownloadhandler=HTTPDownloadHandler, **kw):if not is_botocore_available():raise NotConfigured('missing botocore library')@@ -18,6 +19,8 @@ class S3DownloadHandler:aws_access_key_id = settings['AWS_ACCESS_KEY_ID']if not aws_secret_access_key:aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']+ if not aws_session_token:+ aws_session_token = settings['AWS_SESSION_TOKEN']# If no credentials could be found anywhere,# consider this an anonymous connection request by default;@@ -36,7 +39,7 @@ class S3DownloadHandler:if not self.anon:SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']self._signer = SignerCls(botocore.credentials.Credentials(- aws_access_key_id, aws_secret_access_key))+ aws_access_key_id, aws_secret_access_key, aws_session_token))_http_handler = create_instance(objcls=httpdownloadhandler,diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex bd4808e2b..564c736f2 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -154,13 +154,14 @@ class FileFeedStorage:class S3FeedStorage(BlockingFeedStorage):def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,- feed_options=None):+ feed_options=None, session_token=None):if not is_botocore_available():raise NotConfigured('missing botocore library')u = urlparse(uri)self.bucketname = u.hostnameself.access_key = u.username or access_keyself.secret_key = u.password or secret_key+ self.session_token = session_tokenself.keyname = u.path[1:] # remove first "/"self.acl = aclself.endpoint_url = endpoint_url@@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage):self.s3_client = session.create_client('s3', aws_access_key_id=self.access_key,aws_secret_access_key=self.secret_key,+ aws_session_token=self.session_token,endpoint_url=self.endpoint_url)if feed_options and feed_options.get('overwrite', True) is False:logger.warning('S3 does not support appending to files. To '@@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):uri,access_key=crawler.settings['AWS_ACCESS_KEY_ID'],secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],+ session_token=crawler.settings['AWS_SESSION_TOKEN'],acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,feed_options=feed_options,diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 13ecd4e6c..8766ef66f 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -79,6 +79,7 @@ class FSFilesStore:class S3FilesStore:AWS_ACCESS_KEY_ID = NoneAWS_SECRET_ACCESS_KEY = None+ AWS_SESSION_TOKEN = NoneAWS_ENDPOINT_URL = NoneAWS_REGION_NAME = NoneAWS_USE_SSL = None@@ -98,6 +99,7 @@ class S3FilesStore:'s3',aws_access_key_id=self.AWS_ACCESS_KEY_ID,aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,+ aws_session_token=self.AWS_SESSION_TOKEN,endpoint_url=self.AWS_ENDPOINT_URL,region_name=self.AWS_REGION_NAME,use_ssl=self.AWS_USE_SSL,@@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline):s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']+ s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']s3store.AWS_USE_SSL = settings['AWS_USE_SSL']diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex e3ab23ea5..9c99dc69e 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline):s3store = cls.STORE_SCHEMES['s3']s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']+ s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']s3store.AWS_USE_SSL = settings['AWS_USE_SSL']diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex da0b2c786..389808306 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase):def test_parse_credentials(self):skip_if_no_boto()aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',- 'AWS_SECRET_ACCESS_KEY': 'settings_secret'}+ 'AWS_SECRET_ACCESS_KEY': 'settings_secret',+ 'AWS_SESSION_TOKEN': 'settings_token'}crawler = get_crawler(settings_dict=aws_credentials)# Instantiate with crawlerstorage = S3FeedStorage.from_crawler(@@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase):)self.assertEqual(storage.access_key, 'settings_key')self.assertEqual(storage.secret_key, 'settings_secret')+ self.assertEqual(storage.session_token, 'settings_token')# Instantiate directlystorage = S3FeedStorage('s3://mybucket/export.csv',aws_credentials['AWS_ACCESS_KEY_ID'],- aws_credentials['AWS_SECRET_ACCESS_KEY'])+ aws_credentials['AWS_SECRET_ACCESS_KEY'],+ session_token=aws_credentials['AWS_SESSION_TOKEN'])self.assertEqual(storage.access_key, 'settings_key')self.assertEqual(storage.secret_key, 'settings_secret')+ self.assertEqual(storage.session_token, 'settings_token')# URI priority > settings prioritystorage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',aws_credentials['AWS_ACCESS_KEY_ID'],@@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):class S3FeedStorageWithoutFeedOptions(S3FeedStorage):- def __init__(self, uri, access_key, secret_key, acl, endpoint_url):- super().__init__(uri, access_key, secret_key, acl, endpoint_url)+ def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):+ super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):diff --git a/tox.ini b/tox.iniindex 4c4bbff6e..96050223b 100644--- a/tox.ini+++ b/tox.ini@@ -23,6 +23,7 @@ passenv =S3_TEST_FILE_URIAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEY+ AWS_SESSION_TOKENGCS_TEST_FILE_URIGCS_PROJECT_ID#allow tox virtualenv to upgrade pip/wheel/setuptools