Scrapy, a fast high-level web crawling & scraping framework for Python.
add Headers.to_native_string_dict - useful when interfacing with other libraries
Konstantin Lopuhin1b1092b7d073
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 6d28f866e..0903b84ad 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -85,15 +85,10 @@ class S3DownloadHandler(object):request = request.replace(url=url)elif self._signer is not None:import botocore.awsrequest- from botocore.vendored.requests.structures import CaseInsensitiveDict- print(url, request.headers)awsrequest = botocore.awsrequest.AWSRequest(method=request.method,url='%s://s3.amazonaws.com/%s%s' % (scheme, bucket, path),- # TODO - move to a header method- headers=CaseInsensitiveDict(- (to_unicode(key), to_unicode(b','.join(value)))- for key, value in request.headers.items()),+ headers=request.headers.to_native_string_dict(),data=request.body)self._signer.add_auth(awsrequest)request = request.replace(diff --git a/scrapy/http/headers.py b/scrapy/http/headers.pyindex 13f0f0383..d0c4cd0fb 100644--- a/scrapy/http/headers.py+++ b/scrapy/http/headers.py@@ -1,6 +1,7 @@import sixfrom w3lib.http import headers_dict_to_rawfrom scrapy.utils.datatypes import CaselessDict+from scrapy.utils.python import to_unicodeclass Headers(CaselessDict):@@ -78,6 +79,12 @@ class Headers(CaselessDict):def to_string(self):return headers_dict_to_raw(self)+ def to_native_string_dict(self):+ return CaselessDict(+ (to_unicode(key, encoding=self.encoding),+ to_unicode(b','.join(value), encoding=self.encoding))+ for key, value in self.items())+def __copy__(self):return self.__class__(self)copy = __copy__