Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into response_follow_all

Eugenio Lacuesta48223c107a7c
diff --cc scrapy/http/response/text.py
index 5110b4bd4,37f450e54..b97420345
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@@ -5,18 -5,18 +5,18 @@@ discovering (through HTTP headers) to b
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin
+from contextlib import suppress
import parsel
-from w3lib.encoding import html_to_unicode, resolve_encoding, \
- html_body_declared_encoding, http_content_type_encoding
+import six
+from six.moves.urllib.parse import urljoin
+from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
+ http_content_type_encoding, resolve_encoding)
from w3lib.html import strip_html5_whitespace
-from scrapy.http.request import Request
from scrapy.http.response import Response
- from scrapy.utils.python import memoizemethod_noargs, to_native_str
-from scrapy.utils.response import get_base_url
+ from scrapy.utils.python import memoizemethod_noargs, to_unicode
+from scrapy.utils.response import get_base_url
class TextResponse(Response):
@@@ -88,10 -85,10 +85,10 @@@
def _body_inferred_encoding(self):
if self._cached_benc is None:
- content_type = to_native_str(self.headers.get(b'Content-Type', b''))
+ content_type = to_unicode(self.headers.get(b'Content-Type', b''))
benc, ubody = html_to_unicode(content_type, self.body,
- auto_detect_fun=self._auto_detect_fun,
- default_encoding=self._DEFAULT_ENCODING)
+ auto_detect_fun=self._auto_detect_fun,
+ default_encoding=self._DEFAULT_ENCODING)
self._cached_benc = benc
self._cached_ubody = ubody
return self._cached_benc