Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix assert_aws_environ: check for botocore with boto fallback on PY2
Konstantin Lopuhin19b2910ad145
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.pyindex 51edfd353..1ac2e575f 100644--- a/scrapy/utils/test.py+++ b/scrapy/utils/test.py@@ -5,6 +5,7 @@ This module contains some assorted functions used in testsimport osfrom importlib import import_module+import sixfrom twisted.trial.unittest import SkipTest@@ -12,14 +13,22 @@ def assert_aws_environ():"""Asserts the current environment is suitable for running AWS testsi.Raises SkipTest with the reason if it's not."""- try:- import boto- except ImportError as e:- raise SkipTest(str(e))-+ skip_if_no_boto()if 'AWS_ACCESS_KEY_ID' not in os.environ:raise SkipTest("AWS keys not found")+def skip_if_no_boto():+ try:+ import botocore+ except ImportError:+ if six.PY2:+ try:+ import boto+ except ImportError:+ raise SkipTest('missing botocore or boto library')+ else:+ raise SkipTest('missing botocore library')+def get_crawler(spidercls=None, settings_dict=None):"""Return an unconfigured Crawler object. If settings_dict is given, itwill be used to populate the crawler settings with a project leveldiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex c0342b806..f34a286c2 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -28,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.spiders import Spiderfrom scrapy.http import Requestfrom scrapy.settings import Settings-from scrapy.utils.test import get_crawler+from scrapy.utils.test import get_crawler, skip_if_no_botofrom scrapy.utils.python import to_bytesfrom scrapy.exceptions import NotConfigured@@ -437,22 +437,10 @@ class HttpDownloadHandlerMock(object):return request-class BaseS3TestCase(unittest.TestCase):- try:- import botocore- except ImportError:- if six.PY2:- try:- import boto- except ImportError:- skip = 'missing botocore or boto library'- else:- skip = 'missing botocore library'---class S3AnonTestCase(BaseS3TestCase):+class S3AnonTestCase(unittest.TestCase):def setUp(self):+ skip_if_no_boto()self.s3reqh = S3DownloadHandler(Settings(),httpdownloadhandler=HttpDownloadHandlerMock,#anon=True, # is implicit@@ -469,7 +457,7 @@ class S3AnonTestCase(BaseS3TestCase):httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')-class S3TestCase(BaseS3TestCase):+class S3TestCase(unittest.TestCase):download_handler_cls = S3DownloadHandler# test use same example keys than amazon developer guide@@ -480,6 +468,7 @@ class S3TestCase(BaseS3TestCase):AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'def setUp(self):+ skip_if_no_boto()s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,self.AWS_SECRET_ACCESS_KEY,httpdownloadhandler=HttpDownloadHandlerMock)