Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into response_follow_all
Eugenio Lacuesta48223c107a7c
diff --cc scrapy/http/response/text.pyindex 5110b4bd4,37f450e54..b97420345--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@@ -5,18 -5,18 +5,18 @@@ discovering (through HTTP headers) to bSee documentation in docs/topics/request-response.rst"""-import six-from six.moves.urllib.parse import urljoin+from contextlib import suppressimport parsel-from w3lib.encoding import html_to_unicode, resolve_encoding, \- html_body_declared_encoding, http_content_type_encoding+import six+from six.moves.urllib.parse import urljoin+from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,+ http_content_type_encoding, resolve_encoding)from w3lib.html import strip_html5_whitespace-from scrapy.http.request import Requestfrom scrapy.http.response import Response- from scrapy.utils.python import memoizemethod_noargs, to_native_str-from scrapy.utils.response import get_base_url+ from scrapy.utils.python import memoizemethod_noargs, to_unicode+from scrapy.utils.response import get_base_urlclass TextResponse(Response):@@@ -88,10 -85,10 +85,10 @@@def _body_inferred_encoding(self):if self._cached_benc is None:- content_type = to_native_str(self.headers.get(b'Content-Type', b''))+ content_type = to_unicode(self.headers.get(b'Content-Type', b''))benc, ubody = html_to_unicode(content_type, self.body,- auto_detect_fun=self._auto_detect_fun,- default_encoding=self._DEFAULT_ENCODING)+ auto_detect_fun=self._auto_detect_fun,+ default_encoding=self._DEFAULT_ENCODING)self._cached_benc = bencself._cached_ubody = ubodyreturn self._cached_benc