Scrapy, a fast high-level web crawling & scraping framework for Python.
S3FeedStorageTest: add botocore support, and organize boto/botocore checks
Konstantin Lopuhin3ada45a9bb6a
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex e218a8741..dd7bce2be 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -1,17 +1,13 @@-import sixfrom six.moves.urllib.parse import unquotefrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.python import to_unicode+from scrapy.utils.boto import is_botocorefrom .http import HTTPDownloadHandler-def get_s3_connection():- try:- from boto.s3.connection import S3Connection- except ImportError:- return None+def _get_boto_connection():+ from boto.s3.connection import S3Connectionclass _v19_S3Connection(S3Connection):"""A dummy S3Connection wrapper that doesn't do any synchronous download"""@@ -53,21 +49,9 @@ class S3DownloadHandler(object):self.anon = kw.get('anon')self._signer = None- try:+ if is_botocore():import botocore.authimport botocore.credentials- except ImportError:- if six.PY3:- raise NotConfigured("missing botocore library")- _S3Connection = get_s3_connection()- if _S3Connection is None:- raise NotConfigured("missing botocore or boto library")- try:- self.conn = _S3Connection(- aws_access_key_id, aws_secret_access_key, **kw)- except Exception as ex:- raise NotConfigured(str(ex))- else:kw.pop('anon', None)if kw:raise TypeError('Unexpected keyword arguments: %s' % kw)@@ -75,6 +59,13 @@ class S3DownloadHandler(object):SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']self._signer = SignerCls(botocore.credentials.Credentials(aws_access_key_id, aws_secret_access_key))+ else:+ _S3Connection = _get_boto_connection()+ try:+ self.conn = _S3Connection(+ aws_access_key_id, aws_secret_access_key, **kw)+ except Exception as ex:+ raise NotConfigured(str(ex))self._download_http = httpdownloadhandler(settings).download_requestdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex fce5e251b..3dab2d77e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfiguredfrom scrapy.utils.misc import load_objectfrom scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.python import without_none_values+from scrapy.utils.boto import is_botocorelogger = logging.getLogger(__name__)@@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage):def __init__(self, uri):from scrapy.conf import settings- try:- import boto- except ImportError:- raise NotConfigured- self.connect_s3 = boto.connect_s3u = urlparse(uri)self.bucketname = u.hostnameself.access_key = u.username or settings['AWS_ACCESS_KEY_ID']self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']- self.keyname = u.path+ self.is_botocore = is_botocore()+ self.keyname = u.path[1:] # remove first "/"+ if self.is_botocore:+ import botocore.session+ session = botocore.session.get_session()+ self.s3_client = session.create_client(+ 's3', aws_access_key_id=self.access_key,+ aws_secret_access_key=self.secret_key)+ else:+ import boto+ self.connect_s3 = boto.connect_s3def _store_in_thread(self, file):file.seek(0)- conn = self.connect_s3(self.access_key, self.secret_key)- bucket = conn.get_bucket(self.bucketname, validate=False)- key = bucket.new_key(self.keyname)- key.set_contents_from_file(file)- key.close()+ if self.is_botocore:+ self.s3_client.put_object(+ Bucket=self.bucketname, Key=self.keyname, Body=file)+ else:+ conn = self.connect_s3(self.access_key, self.secret_key)+ bucket = conn.get_bucket(self.bucketname, validate=False)+ key = bucket.new_key(self.keyname)+ key.set_contents_from_file(file)+ key.close()class FTPFeedStorage(BlockingFeedStorage):diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.pynew file mode 100644index 000000000..fd6b22b88--- /dev/null+++ b/scrapy/utils/boto.py@@ -0,0 +1,20 @@+"""Boto/botocore helpers"""++import six++from scrapy.exceptions import NotConfigured+++def is_botocore():+ try:+ import botocore+ return True+ except ImportError:+ if six.PY2:+ try:+ import boto+ return False+ except ImportError:+ raise NotConfigured('missing botocore or boto library')+ else:+ raise NotConfigured('missing botocore library')diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 1ac2e575f..d2f7c0ae4 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -5,9 +5,11 @@ This module contains some assorted functions used in testsimport osfrom importlib import import_module-import sixfrom twisted.trial.unittest import SkipTest+from scrapy.exceptions import NotConfigured+from scrapy.utils.boto import is_botocore+def assert_aws_environ():"""Asserts the current environment is suitable for running AWS testsi.@@ -19,15 +21,9 @@ def assert_aws_environ():def skip_if_no_boto():try:- import botocore- except ImportError:- if six.PY2:- try:- import boto- except ImportError:- raise SkipTest('missing botocore or boto library')- else:- raise SkipTest('missing botocore library')+ is_botocore()+ except NotConfigured as e:+ raise SkipTest(e.message)def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 8015b0320..beb800fb5 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import ()from scrapy.utils.test import assert_aws_environfrom scrapy.utils.python import to_native_str+from scrapy.utils.boto import is_botocoreclass FileFeedStorageTest(unittest.TestCase):@@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase):uri = os.environ.get('FEEDTEST_S3_URI')if not uri:raise unittest.SkipTest("No S3 URI available for testing")- from boto import connect_s3storage = S3FeedStorage(uri)verifyObject(IFeedStorage, storage)file = storage.open(scrapy.Spider("default"))file.write("content")yield storage.store(file)u = urlparse(uri)- bucket = connect_s3().get_bucket(u.hostname, validate=False)- key = bucket.get_key(u.path)- self.assertEqual(key.get_contents_as_string(), "content")- bucket.delete_key(u.path)+ content = self._get_content_and_delete(u.hostname, u.path[1:])+ self.assertEqual(content, "content")++ def _get_content_and_delete(self, bucket, path):+ if is_botocore():+ import botocore.session+ session = botocore.session.get_session()+ client = session.create_client('s3')+ key = client.get_object(Bucket=bucket, Key=path)+ content = key['Body'].read()+ client.delete_object(Bucket=bucket, Key=path)+ else:+ from boto import connect_s3+ bucket = connect_s3().get_bucket(bucket, validate=False)+ key = bucket.get_key(path)+ content = key.get_contents_as_string()+ bucket.delete_key(path)+ return contentclass StdoutFeedStorageTest(unittest.TestCase):