Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add Headers.to_native_string_dict - useful when interfacing with other libraries

Konstantin Lopuhin1b1092b7d073
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 6d28f866e..0903b84ad 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -85,15 +85,10 @@ class S3DownloadHandler(object):
request = request.replace(url=url)
elif self._signer is not None:
import botocore.awsrequest
- from botocore.vendored.requests.structures import CaseInsensitiveDict
- print(url, request.headers)
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),
- # TODO - move to a header method
- headers=CaseInsensitiveDict(
- (to_unicode(key), to_unicode(b','.join(value)))
- for key, value in request.headers.items()),
+ headers=request.headers.to_native_string_dict(),
data=request.body)
self._signer.add_auth(awsrequest)
request = request.replace(
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 13f0f0383..d0c4cd0fb 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,6 +1,7 @@
import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.python import to_unicode
class Headers(CaselessDict):
@@ -78,6 +79,12 @@ class Headers(CaselessDict):
def to_string(self):
return headers_dict_to_raw(self)
+ def to_native_string_dict(self):
+ return CaselessDict(
+ (to_unicode(key, encoding=self.encoding),
+ to_unicode(b','.join(value), encoding=self.encoding))
+ for key, value in self.items())
+
def __copy__(self):
return self.__class__(self)
copy = __copy__