Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

S3FeedStorageTest: add botocore support, and organize boto/botocore checks

Konstantin Lopuhin3ada45a9bb6a
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index e218a8741..dd7bce2be 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,17 +1,13 @@
-import six
from six.moves.urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_unicode
+from scrapy.utils.boto import is_botocore
from .http import HTTPDownloadHandler
-def get_s3_connection():
- try:
- from boto.s3.connection import S3Connection
- except ImportError:
- return None
+def _get_boto_connection():
+ from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
@@ -53,21 +49,9 @@ class S3DownloadHandler(object):
self.anon = kw.get('anon')
self._signer = None
- try:
+ if is_botocore():
import botocore.auth
import botocore.credentials
- except ImportError:
- if six.PY3:
- raise NotConfigured("missing botocore library")
- _S3Connection = get_s3_connection()
- if _S3Connection is None:
- raise NotConfigured("missing botocore or boto library")
- try:
- self.conn = _S3Connection(
- aws_access_key_id, aws_secret_access_key, **kw)
- except Exception as ex:
- raise NotConfigured(str(ex))
- else:
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
@@ -75,6 +59,13 @@ class S3DownloadHandler(object):
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
+ else:
+ _S3Connection = _get_boto_connection()
+ try:
+ self.conn = _S3Connection(
+ aws_access_key_id, aws_secret_access_key, **kw)
+ except Exception as ex:
+ raise NotConfigured(str(ex))
self._download_http = httpdownloadhandler(settings).download_request
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fce5e251b..3dab2d77e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import load_object
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import without_none_values
+from scrapy.utils.boto import is_botocore
logger = logging.getLogger(__name__)
@@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri):
from scrapy.conf import settings
- try:
- import boto
- except ImportError:
- raise NotConfigured
- self.connect_s3 = boto.connect_s3
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
- self.keyname = u.path
+ self.is_botocore = is_botocore()
+ self.keyname = u.path[1:] # remove first "/"
+ if self.is_botocore:
+ import botocore.session
+ session = botocore.session.get_session()
+ self.s3_client = session.create_client(
+ 's3', aws_access_key_id=self.access_key,
+ aws_secret_access_key=self.secret_key)
+ else:
+ import boto
+ self.connect_s3 = boto.connect_s3
def _store_in_thread(self, file):
file.seek(0)
- conn = self.connect_s3(self.access_key, self.secret_key)
- bucket = conn.get_bucket(self.bucketname, validate=False)
- key = bucket.new_key(self.keyname)
- key.set_contents_from_file(file)
- key.close()
+ if self.is_botocore:
+ self.s3_client.put_object(
+ Bucket=self.bucketname, Key=self.keyname, Body=file)
+ else:
+ conn = self.connect_s3(self.access_key, self.secret_key)
+ bucket = conn.get_bucket(self.bucketname, validate=False)
+ key = bucket.new_key(self.keyname)
+ key.set_contents_from_file(file)
+ key.close()
class FTPFeedStorage(BlockingFeedStorage):
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
new file mode 100644
index 000000000..fd6b22b88
--- /dev/null
+++ b/scrapy/utils/boto.py
@@ -0,0 +1,20 @@
+"""Boto/botocore helpers"""
+
+import six
+
+from scrapy.exceptions import NotConfigured
+
+
+def is_botocore():
+ try:
+ import botocore
+ return True
+ except ImportError:
+ if six.PY2:
+ try:
+ import boto
+ return False
+ except ImportError:
+ raise NotConfigured('missing botocore or boto library')
+ else:
+ raise NotConfigured('missing botocore library')
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 1ac2e575f..d2f7c0ae4 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -5,9 +5,11 @@ This module contains some assorted functions used in tests
import os
from importlib import import_module
-import six
from twisted.trial.unittest import SkipTest
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.boto import is_botocore
+
def assert_aws_environ():
"""Asserts the current environment is suitable for running AWS testsi.
@@ -19,15 +21,9 @@ def assert_aws_environ():
def skip_if_no_boto():
try:
- import botocore
- except ImportError:
- if six.PY2:
- try:
- import boto
- except ImportError:
- raise SkipTest('missing botocore or boto library')
- else:
- raise SkipTest('missing botocore library')
+ is_botocore()
+ except NotConfigured as e:
+ raise SkipTest(e.message)
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 8015b0320..beb800fb5 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import (
)
from scrapy.utils.test import assert_aws_environ
from scrapy.utils.python import to_native_str
+from scrapy.utils.boto import is_botocore
class FileFeedStorageTest(unittest.TestCase):
@@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase):
uri = os.environ.get('FEEDTEST_S3_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
- from boto import connect_s3
storage = S3FeedStorage(uri)
verifyObject(IFeedStorage, storage)
file = storage.open(scrapy.Spider("default"))
file.write("content")
yield storage.store(file)
u = urlparse(uri)
- bucket = connect_s3().get_bucket(u.hostname, validate=False)
- key = bucket.get_key(u.path)
- self.assertEqual(key.get_contents_as_string(), "content")
- bucket.delete_key(u.path)
+ content = self._get_content_and_delete(u.hostname, u.path[1:])
+ self.assertEqual(content, "content")
+
+ def _get_content_and_delete(self, bucket, path):
+ if is_botocore():
+ import botocore.session
+ session = botocore.session.get_session()
+ client = session.create_client('s3')
+ key = client.get_object(Bucket=bucket, Key=path)
+ content = key['Body'].read()
+ client.delete_object(Bucket=bucket, Key=path)
+ else:
+ from boto import connect_s3
+ bucket = connect_s3().get_bucket(bucket, validate=False)
+ key = bucket.get_key(path)
+ content = key.get_contents_as_string()
+ bucket.delete_key(path)
+ return content
class StdoutFeedStorageTest(unittest.TestCase):