Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support headers in S3FilesStore.persist_file for botocore

Konstantin Lopuhind61fbcc8b5fe
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c757b0a3f..45ceddcbb 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -29,6 +29,7 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import to_bytes
from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
+from scrapy.utils.datatypes import CaselessDict
logger = logging.getLogger(__name__)
@@ -137,13 +138,17 @@ class S3FilesStore(object):
key_name = '%s%s' % (self.prefix, path)
buf.seek(0)
if self.is_botocore:
+ extra = self._headers_to_botocore_kwargs(self.HEADERS)
+ if headers:
+ extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in six.iteritems(meta)},
- ACL=self.POLICY)
+ ACL=self.POLICY,
+ **extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
@@ -157,6 +162,35 @@ class S3FilesStore(object):
k.set_contents_from_string, buf.getvalue(),
headers=h, policy=self.POLICY)
+ def _headers_to_botocore_kwargs(self, headers):
+ """ Convert headers to botocore keyword agruments.
+ """
+ # This is required while we need to support both boto and botocore.
+ mapping = CaselessDict({
+ 'Content-Type': 'ContentType',
+ 'Cache-Control': 'CacheControl',
+ 'Content-Disposition': 'ContentDisposition',
+ 'Content-Encoding': 'ContentEncoding',
+ 'Content-Language': 'ContentLanguage',
+ 'Content-Length': 'ContentLength',
+ 'Content-MD5': 'ContentMD5',
+ 'Expires': 'Expires',
+ 'X-Amz-Grant-Full-Control': 'GrantFullControl',
+ 'X-Amz-Grant-Read': 'GrantRead',
+ 'X-Amz-Grant-Read-ACP': 'GrantReadACP',
+ 'X-Amz-Grant-Write-ACP': 'GrantWriteACP',
+ })
+ extra = {}
+ for key, value in six.iteritems(headers):
+ try:
+ kwarg = mapping[key]
+ except KeyError:
+ raise TypeError(
+ 'Header "%s" is not supported by botocore' % key)
+ else:
+ extra[kwarg] = value
+ return extra
+
class FilesPipeline(MediaPipeline):
"""Abstract pipeline that implement the file downloading
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 43abd64a0..bf66a8cbe 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -26,7 +26,7 @@ def skip_if_no_boto():
except NotConfigured as e:
raise SkipTest(e.message)
-def get_s3_content_and_delete(bucket, path):
+def get_s3_content_and_delete(bucket, path, with_key=False):
""" Get content from s3 key, and delete key afterwards.
"""
if is_botocore():
@@ -43,7 +43,7 @@ def get_s3_content_and_delete(bucket, path):
key = bucket.get_key(path)
content = key.get_contents_as_string()
bucket.delete_key(path)
- return content
+ return (content, key) if with_key else content
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index e445d9989..77e75d5ac 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -16,6 +16,7 @@ from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.utils.python import to_bytes
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+from scrapy.utils.boto import is_botocore
from tests import mock
@@ -194,14 +195,27 @@ class TestS3FilesStore(unittest.TestCase):
meta = {'foo': 'bar'}
path = ''
store = S3FilesStore(uri)
- yield store.persist_file(path, buf, info=None, meta=meta)
+ yield store.persist_file(
+ path, buf, info=None, meta=meta,
+ headers={'Content-Type': 'image/png'})
s = yield store.stat_file(path, info=None)
self.assertIn('last_modified', s)
self.assertIn('checksum', s)
self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
u = urlparse(uri)
- content = get_s3_content_and_delete(u.hostname, u.path[1:])
+ content, key = get_s3_content_and_delete(
+ u.hostname, u.path[1:], with_key=True)
self.assertEqual(content, data)
+ if is_botocore():
+ self.assertEqual(key['Metadata'], {'foo': 'bar'})
+ self.assertEqual(
+ key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
+ self.assertEqual(key['ContentType'], 'image/png')
+ else:
+ self.assertEqual(key.metadata, {'foo': 'bar'})
+ self.assertEqual(
+ key.cache_control, S3FilesStore.HEADERS['Cache-Control'])
+ self.assertEqual(key.content_type, 'image/png')
class ItemWithFiles(Item):