Scrapy, a fast high-level web crawling & scraping framework for Python.
response.text. Fixes GH-1729.
Mikhail Korobov4bcbb77bcc7d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex ea64d1599..2e92961a9 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -445,10 +445,10 @@ Response objects.. attribute:: Response.body- A str containing the body of this Response. Keep in mind that Response.body- is always a str. If you want the unicode version use- :meth:`TextResponse.body_as_unicode` (only available in- :class:`TextResponse` and subclasses).+ The body of this Response. Keep in mind that Response.body+ is always a bytes object. If you want the unicode version use+ :attr:`TextResponse.txt` (only available in :class:`TextResponse`+ and subclasses).This attribute is read-only. To change the body of a Response use:meth:`replace`.@@ -542,6 +542,21 @@ TextResponse objects:class:`TextResponse` objects support the following attributes in additionto the standard :class:`Response` ones:+ .. attribute:: TextResponse.text++ Response body, as unicode.++ The same as ``response.body.decode(response.encoding)``, but the+ result is cached after the first call, so you can access+ ``response.text`` multiple times without extra overhead.++ .. note::++ ``unicode(response.body)`` is not a correct way to convert response+ body to unicode: you would be using the system default encoding+ (typically `ascii`) instead of the response encoding.++.. attribute:: TextResponse.encodingA string with the encoding of this response. The encoding is resolved by@@ -568,20 +583,6 @@ TextResponse objects:class:`TextResponse` objects support the following methods in addition tothe standard :class:`Response` ones:- .. method:: TextResponse.body_as_unicode()-- Returns the body of the response as unicode. This is equivalent to::-- response.body.decode(response.encoding)-- But **not** equivalent to::-- unicode(response.body)-- Since, in the latter case, you would be using the system default encoding- (typically `ascii`) to convert the body to unicode, instead of the response- encoding.-.. method:: TextResponse.xpath(query)A shortcut to ``TextResponse.selector.xpath(query)``::@@ -594,6 +595,11 @@ TextResponse objectsresponse.css('p')+ .. method:: TextResponse.body_as_unicode()++ The same as :attr:`text`, but available as a method. This method is+ kept for backwards compatibility; please prefer ``response.text``.+HtmlResponse objects--------------------diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.pyindex 6b543b823..da373eca2 100644--- a/scrapy/downloadermiddlewares/ajaxcrawl.py+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py@@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object):Return True if a page without hash fragment could be "AJAX crawlable"according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started."""- body = response.body_as_unicode()[:self.lookup_bytes]+ body = response.text[:self.lookup_bytes]return _has_ajaxcrawlable_meta(body)diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex c061c2407..d4a33dc36 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object):def _parse_robots(self, response, netloc):rp = robotparser.RobotFileParser(response.url)body = ''- if hasattr(response, 'body_as_unicode'):- body = response.body_as_unicode()+ if hasattr(response, 'text'):+ body = response.textelse: # last effort trytry:body = response.body.decode('utf-8')diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 5501634d3..2862dc096 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -64,8 +64,8 @@ def _urlencode(seq, enc):def _get_form(response, formname, formid, formnumber, formxpath):"""Find the form element """- text = response.body_as_unicode()- root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))+ root = create_root_node(response.text, lxml.html.HTMLParser,+ base_url=get_base_url(response))forms = root.xpath('//form')if not forms:raise ValueError("No <form> element found in %s" % response)diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 1c416bf82..9c667ab7e 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -67,6 +67,11 @@ class TextResponse(Response):self._cached_ubody = html_to_unicode(charset, self.body)[1]return self._cached_ubody+ @property+ def text(self):+ """ Body as unicode """+ return self.body_as_unicode()+def urljoin(self, url):"""Join this Response's url with a possible relative url to form anabsolute interpretation of the latter."""diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.pyindex 5d77f7624..15f3d26df 100644--- a/scrapy/selector/unified.py+++ b/scrapy/selector/unified.py@@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref):response = _response_from_text(text, st)if response is not None:- text = response.body_as_unicode()+ text = response.textkwargs.setdefault('base_url', response.url)self.response = responsediff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.pyindex b0688791e..73857b410 100644--- a/scrapy/utils/iterators.py+++ b/scrapy/utils/iterators.py@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):if not unicode:return obj.bodyelif isinstance(obj, TextResponse):- return obj.body_as_unicode()+ return obj.textelse:return obj.body.decode('utf-8')elif isinstance(obj, six.text_type):diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex c4ad52f14..73db2641e 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary()def get_base_url(response):"""Return the base url of the given response, joined with the response url"""if response not in _baseurl_cache:- text = response.body_as_unicode()[0:4096]+ text = response.text[0:4096]_baseurl_cache[response] = html.get_base_url(text, response.url,response.encoding)return _baseurl_cache[response]@@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary()def get_meta_refresh(response):"""Parse the http-equiv refrsh parameter from the given response"""if response not in _metaref_cache:- text = response.body_as_unicode()[0:4096]+ text = response.text[0:4096]text = _noscript_re.sub(u'', text)text = _script_re.sub(u'', text)_metaref_cache[response] = html.get_meta_refresh(text, response.url,diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 9f2c02bff..baf6ef1bf 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -55,12 +55,11 @@ class TestSpider(Spider):def parse_item(self, response):item = self.item_cls()- body = response.body_as_unicode()- m = self.name_re.search(body)+ m = self.name_re.search(response.text)if m:item['name'] = m.group(1)item['url'] = response.url- m = self.price_re.search(body)+ m = self.price_re.search(response.text)if m:item['price'] = m.group(1)return itemdiff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 710a5b29d..c7f36687a 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase):body_bytes = bodyassert isinstance(response.body, bytes)+ assert isinstance(response.text, six.text_type)self._assert_response_encoding(response, encoding)self.assertEqual(response.body, body_bytes)self.assertEqual(response.body_as_unicode(), body_unicode)+ self.assertEqual(response.text, body_unicode)def _assert_response_encoding(self, response, encoding):self.assertEqual(response.encoding, resolve_encoding(encoding))@@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest):self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))self.assertEqual(r1.body_as_unicode(), unicode_string)+ # check response.text+ self.assertTrue(isinstance(r1.text, six.text_type))+ self.assertEqual(r1.text, unicode_string)+def test_encoding(self):r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")@@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest):headers={"Content-type": ["text/html; charset=utf-8"]},body=b"\xef\xbb\xbfWORD\xe3\xab")self.assertEqual(r6.encoding, 'utf-8')- self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')+ self.assertEqual(r6.text, u'WORD\ufffd\ufffd')def test_bom_is_removed_from_body(self):# Inferring encoding from body also cache decoded body as sideeffect,# this test tries to ensure that calling response.encoding and- # response.body_as_unicode() in indistint order doesn't affect final+ # response.text in indistint order doesn't affect final# values for encoding and decoded body.url = 'http://example.com'body = b"\xef\xbb\xbfWORD"@@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest):# Test response without content-type and BOM encodingresponse = self.response_class(url, body=body)self.assertEqual(response.encoding, 'utf-8')- self.assertEqual(response.body_as_unicode(), u'WORD')+ self.assertEqual(response.text, u'WORD')response = self.response_class(url, body=body)- self.assertEqual(response.body_as_unicode(), u'WORD')+ self.assertEqual(response.text, u'WORD')self.assertEqual(response.encoding, 'utf-8')# Body caching sideeffect isn't triggered when encoding is declared in@@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest):# bodyresponse = self.response_class(url, headers=headers, body=body)self.assertEqual(response.encoding, 'utf-8')- self.assertEqual(response.body_as_unicode(), u'WORD')+ self.assertEqual(response.text, u'WORD')response = self.response_class(url, headers=headers, body=body)- self.assertEqual(response.body_as_unicode(), u'WORD')+ self.assertEqual(response.text, u'WORD')self.assertEqual(response.encoding, 'utf-8')def test_replace_wrong_encoding(self):@@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest):r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')# XXX: Policy for replacing invalid chars may suffer minor variations# but it should always contain the unicode replacement char (u'\ufffd')- assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())- assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode())- assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode())+ assert u'\ufffd' in r.text, repr(r.text)+ assert u'PREFIX' in r.text, repr(r.text)+ assert u'SUFFIX' in r.text, repr(r.text)# Do not destroy html tags due to encoding bugsr = self.response_class("http://example.com", encoding='utf-8', \body=b'\xf0<span>value</span>')- assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())+ assert u'<span>value</span>' in r.text, repr(r.text)# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse- #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX')- #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())+ #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')+ #assert u'\ufffd' in r.text, repr(r.text)def test_selector(self):body = b"<html><head><title>Some page</title><body></body></html>"