Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix assert_aws_environ: check for botocore with boto fallback on PY2

Konstantin Lopuhin19b2910ad145
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 51edfd353..1ac2e575f 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -5,6 +5,7 @@ This module contains some assorted functions used in tests
import os
from importlib import import_module
+import six
from twisted.trial.unittest import SkipTest
@@ -12,14 +13,22 @@ def assert_aws_environ():
"""Asserts the current environment is suitable for running AWS testsi.
Raises SkipTest with the reason if it's not.
"""
- try:
- import boto
- except ImportError as e:
- raise SkipTest(str(e))
-
+ skip_if_no_boto()
if 'AWS_ACCESS_KEY_ID' not in os.environ:
raise SkipTest("AWS keys not found")
+def skip_if_no_boto():
+ try:
+ import botocore
+ except ImportError:
+ if six.PY2:
+ try:
+ import boto
+ except ImportError:
+ raise SkipTest('missing botocore or boto library')
+ else:
+ raise SkipTest('missing botocore library')
+
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
will be used to populate the crawler settings with a project level
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c0342b806..f34a286c2 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -28,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
from scrapy.exceptions import NotConfigured
@@ -437,22 +437,10 @@ class HttpDownloadHandlerMock(object):
return request
-class BaseS3TestCase(unittest.TestCase):
- try:
- import botocore
- except ImportError:
- if six.PY2:
- try:
- import boto
- except ImportError:
- skip = 'missing botocore or boto library'
- else:
- skip = 'missing botocore library'
-
-
-class S3AnonTestCase(BaseS3TestCase):
+class S3AnonTestCase(unittest.TestCase):
def setUp(self):
+ skip_if_no_boto()
self.s3reqh = S3DownloadHandler(Settings(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
@@ -469,7 +457,7 @@ class S3AnonTestCase(BaseS3TestCase):
httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
-class S3TestCase(BaseS3TestCase):
+class S3TestCase(unittest.TestCase):
download_handler_cls = S3DownloadHandler
# test use same example keys than amazon developer guide
@@ -480,6 +468,7 @@ class S3TestCase(BaseS3TestCase):
AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
def setUp(self):
+ skip_if_no_boto()
s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock)