Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Switch from `is_boto3_available()` to `IS_BOTO3_AVAILABLE` var

jazzthief4ebc08ef1042
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4f0a946de..83849ca13 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -21,7 +21,7 @@ from zope.interface import Interface, implementer
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.utils.boto import is_boto3_available, is_botocore_available
+from scrapy.utils.boto import is_botocore_available
from scrapy.utils.conf import feed_complete_default_values_from_settings
from scrapy.utils.ftp import ftp_store_file
from scrapy.utils.log import failure_to_exc_info
@@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
+try:
+ import boto3 # noqa: F401
+
+ IS_BOTO3_AVAILABLE = True
+except ImportError:
+ IS_BOTO3_AVAILABLE = False
+
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
@@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
- self._using_boto3 = is_boto3_available()
- if self._using_boto3:
+ if IS_BOTO3_AVAILABLE:
import boto3.session
session = boto3.session.Session()
@@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file):
file.seek(0)
- if self._using_boto3:
+ if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 7b18b6bcf..085ee7d25 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -8,12 +8,3 @@ def is_botocore_available():
return True
except ImportError:
return False
-
-
-def is_boto3_available():
- try:
- import boto3 # noqa: F401
-
- return True
- except ImportError:
- return False
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 2e350df65..7df3e6dd3 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -35,6 +35,7 @@ import scrapy
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import (
+ IS_BOTO3_AVAILABLE,
BlockingFeedStorage,
FeedExporter,
FileFeedStorage,
@@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import (
S3FeedStorage,
StdoutFeedStorage,
_FeedSlot,
- is_boto3_available,
)
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
@@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase):
file = mock.MagicMock()
- if is_boto3_available():
+ if IS_BOTO3_AVAILABLE:
storage.s3_client = mock.MagicMock()
yield storage.store(file)
self.assertEqual(
@@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
- if is_boto3_available():
+ if IS_BOTO3_AVAILABLE:
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
@@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
- if is_boto3_available():
+ if IS_BOTO3_AVAILABLE:
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
else:
acl = storage.s3_client.put_object.call_args[1]["ACL"]