Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Requested changes

Eugenio Lacuesta98d74d1083be
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 067887d94..7c7db387e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -96,21 +96,22 @@ class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None):
# BEGIN Backwards compatibility for initialising without keys (and
# without using from_crawler)
- from scrapy.conf import settings
no_defaults = access_key is None and secret_key is None
- if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or
- 'AWS_SECRET_ACCESS_KEY' in settings):
- import warnings
- from scrapy.exceptions import ScrapyDeprecationWarning
- warnings.warn(
- "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
- "without AWS keys is deprecated. Please supply credentials or "
- "use the `from_crawler()` constructor.",
- category=ScrapyDeprecationWarning,
- stacklevel=2
- )
- access_key = settings['AWS_ACCESS_KEY_ID']
- secret_key = settings['AWS_SECRET_ACCESS_KEY']
+ if no_defaults:
+ from scrapy.conf import settings
+ if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
+ import warnings
+ from scrapy.exceptions import ScrapyDeprecationWarning
+ warnings.warn(
+ "Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
+ "without AWS keys is deprecated. Please supply credentials or "
+ "use the `from_crawler()` constructor.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2
+ )
+ access_key = settings['AWS_ACCESS_KEY_ID']
+ secret_key = settings['AWS_SECRET_ACCESS_KEY']
+ # END Backwards compatibility
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 8eb1aabb5..5ccfdcd72 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -118,6 +118,7 @@ def rel_has_nofollow(rel):
"""Return True if link rel attribute has nofollow type"""
return True if rel is not None and 'nofollow' in rel.split() else False
+
def create_instance(objcls, settings, crawler, *args, **kwargs):
"""Construct a class instance using its ``from_crawler`` or
``from_settings`` constructors, if available.
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eeb1bc2a4..380ed971b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -173,7 +173,9 @@ class S3FeedStorageTest(unittest.TestCase):
uri = os.environ.get('S3_TEST_FILE_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
- storage = S3FeedStorage(uri, Settings())
+ access_key = os.environ.get('AWS_ACCESS_KEY_ID')
+ secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
+ storage = S3FeedStorage(uri, access_key, secret_key)
verifyObject(IFeedStorage, storage)
file = storage.open(scrapy.Spider("default"))
expected_content = b"content: \xe2\x98\x83"