Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add support for temporary security credential in AWS auth

laggardkernel7e4321f201a7
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 216a8bc52..af60de716 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -135,7 +135,7 @@ Here are some examples to illustrate:
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
-.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
+.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
@@ -200,6 +200,7 @@ passed through the following settings:
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
+- :setting:`AWS_SESSION_TOKEN` (Optional)
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
@@ -357,7 +358,7 @@ For instance::
'item_export_kwargs': {
'export_empty_fields': True,
},
- },
+ },
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 1290b4a5e..58daafa6f 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -204,6 +204,20 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+.. setting:: AWS_SESSION_TOKEN
+
+AWS_SESSION_TOKEN
+-----------------
+
+Default: ``None`` (Optional)
+
+The AWS security token used by code that requires access to `Amazon Web services`_,
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+
+The security token is only required by a *temporary security credentials*.
+Using of temporary security credentials is discouraged cause the credentials
+are short term. It may expires before the scraping is done.
+
.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 31f1be31a..51ca1ed5e 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -10,6 +10,7 @@ class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
+ aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
@@ -18,6 +19,8 @@ class S3DownloadHandler:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+ if not aws_session_token:
+ aws_session_token = settings['AWS_SESSION_TOKEN']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
@@ -36,7 +39,7 @@ class S3DownloadHandler:
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
- aws_access_key_id, aws_secret_access_key))
+ aws_access_key_id, aws_secret_access_key, aws_session_token))
_http_handler = create_instance(
objcls=httpdownloadhandler,
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index bd4808e2b..564c736f2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -154,13 +154,14 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
- feed_options=None):
+ feed_options=None, session_token=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
+ self.session_token = session_token
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
@@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage):
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
+ aws_session_token=self.session_token,
endpoint_url=self.endpoint_url)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
@@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):
uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
+ session_token=crawler.settings['AWS_SESSION_TOKEN'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
feed_options=feed_options,
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 13ecd4e6c..8766ef66f 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -79,6 +79,7 @@ class FSFilesStore:
class S3FilesStore:
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
+ AWS_SESSION_TOKEN = None
AWS_ENDPOINT_URL = None
AWS_REGION_NAME = None
AWS_USE_SSL = None
@@ -98,6 +99,7 @@ class S3FilesStore:
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
+ aws_session_token=self.AWS_SESSION_TOKEN,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
@@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+ s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e3ab23ea5..9c99dc69e 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+ s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
s3store.AWS_USE_SSL = settings['AWS_USE_SSL']
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index da0b2c786..389808306 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase):
def test_parse_credentials(self):
skip_if_no_boto()
aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
- 'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
+ 'AWS_SECRET_ACCESS_KEY': 'settings_secret',
+ 'AWS_SESSION_TOKEN': 'settings_token'}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
storage = S3FeedStorage.from_crawler(
@@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase):
)
self.assertEqual(storage.access_key, 'settings_key')
self.assertEqual(storage.secret_key, 'settings_secret')
+ self.assertEqual(storage.session_token, 'settings_token')
# Instantiate directly
storage = S3FeedStorage('s3://mybucket/export.csv',
aws_credentials['AWS_ACCESS_KEY_ID'],
- aws_credentials['AWS_SECRET_ACCESS_KEY'])
+ aws_credentials['AWS_SECRET_ACCESS_KEY'],
+ session_token=aws_credentials['AWS_SESSION_TOKEN'])
self.assertEqual(storage.access_key, 'settings_key')
self.assertEqual(storage.secret_key, 'settings_secret')
+ self.assertEqual(storage.session_token, 'settings_token')
# URI priority > settings priority
storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',
aws_credentials['AWS_ACCESS_KEY_ID'],
@@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
- def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
- super().__init__(uri, access_key, secret_key, acl, endpoint_url)
+ def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
+ super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
diff --git a/tox.ini b/tox.ini
index 4c4bbff6e..96050223b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,7 @@ passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
+ AWS_SESSION_TOKEN
GCS_TEST_FILE_URI
GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools