Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

response.text. Fixes GH-1729.

Mikhail Korobov4bcbb77bcc7d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ea64d1599..2e92961a9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -445,10 +445,10 @@ Response objects
.. attribute:: Response.body
- A str containing the body of this Response. Keep in mind that Response.body
- is always a str. If you want the unicode version use
- :meth:`TextResponse.body_as_unicode` (only available in
- :class:`TextResponse` and subclasses).
+ The body of this Response. Keep in mind that Response.body
+ is always a bytes object. If you want the unicode version use
+ :attr:`TextResponse.txt` (only available in :class:`TextResponse`
+ and subclasses).
This attribute is read-only. To change the body of a Response use
:meth:`replace`.
@@ -542,6 +542,21 @@ TextResponse objects
:class:`TextResponse` objects support the following attributes in addition
to the standard :class:`Response` ones:
+ .. attribute:: TextResponse.text
+
+ Response body, as unicode.
+
+ The same as ``response.body.decode(response.encoding)``, but the
+ result is cached after the first call, so you can access
+ ``response.text`` multiple times without extra overhead.
+
+ .. note::
+
+ ``unicode(response.body)`` is not a correct way to convert response
+ body to unicode: you would be using the system default encoding
+ (typically `ascii`) instead of the response encoding.
+
+
.. attribute:: TextResponse.encoding
A string with the encoding of this response. The encoding is resolved by
@@ -568,20 +583,6 @@ TextResponse objects
:class:`TextResponse` objects support the following methods in addition to
the standard :class:`Response` ones:
- .. method:: TextResponse.body_as_unicode()
-
- Returns the body of the response as unicode. This is equivalent to::
-
- response.body.decode(response.encoding)
-
- But **not** equivalent to::
-
- unicode(response.body)
-
- Since, in the latter case, you would be using the system default encoding
- (typically `ascii`) to convert the body to unicode, instead of the response
- encoding.
-
.. method:: TextResponse.xpath(query)
A shortcut to ``TextResponse.selector.xpath(query)``::
@@ -594,6 +595,11 @@ TextResponse objects
response.css('p')
+ .. method:: TextResponse.body_as_unicode()
+
+ The same as :attr:`text`, but available as a method. This method is
+ kept for backwards compatibility; please prefer ``response.text``.
+
HtmlResponse objects
--------------------
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 6b543b823..da373eca2 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -63,7 +63,7 @@ class AjaxCrawlMiddleware(object):
Return True if a page without hash fragment could be "AJAX crawlable"
according to https://developers.google.com/webmasters/ajax-crawling/docs/getting-started.
"""
- body = response.body_as_unicode()[:self.lookup_bytes]
+ body = response.text[:self.lookup_bytes]
return _has_ajaxcrawlable_meta(body)
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index c061c2407..d4a33dc36 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -83,8 +83,8 @@ class RobotsTxtMiddleware(object):
def _parse_robots(self, response, netloc):
rp = robotparser.RobotFileParser(response.url)
body = ''
- if hasattr(response, 'body_as_unicode'):
- body = response.body_as_unicode()
+ if hasattr(response, 'text'):
+ body = response.text
else: # last effort try
try:
body = response.body.decode('utf-8')
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 5501634d3..2862dc096 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -64,8 +64,8 @@ def _urlencode(seq, enc):
def _get_form(response, formname, formid, formnumber, formxpath):
"""Find the form element """
- text = response.body_as_unicode()
- root = create_root_node(text, lxml.html.HTMLParser, base_url=get_base_url(response))
+ root = create_root_node(response.text, lxml.html.HTMLParser,
+ base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError("No <form> element found in %s" % response)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 1c416bf82..9c667ab7e 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -67,6 +67,11 @@ class TextResponse(Response):
self._cached_ubody = html_to_unicode(charset, self.body)[1]
return self._cached_ubody
+ @property
+ def text(self):
+ """ Body as unicode """
+ return self.body_as_unicode()
+
def urljoin(self, url):
"""Join this Response's url with a possible relative url to form an
absolute interpretation of the latter."""
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 5d77f7624..15f3d26df 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -60,7 +60,7 @@ class Selector(_ParselSelector, object_ref):
response = _response_from_text(text, st)
if response is not None:
- text = response.body_as_unicode()
+ text = response.text
kwargs.setdefault('base_url', response.url)
self.response = response
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index b0688791e..73857b410 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
if not unicode:
return obj.body
elif isinstance(obj, TextResponse):
- return obj.body_as_unicode()
+ return obj.text
else:
return obj.body.decode('utf-8')
elif isinstance(obj, six.text_type):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index c4ad52f14..73db2641e 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -25,7 +25,7 @@ _baseurl_cache = weakref.WeakKeyDictionary()
def get_base_url(response):
"""Return the base url of the given response, joined with the response url"""
if response not in _baseurl_cache:
- text = response.body_as_unicode()[0:4096]
+ text = response.text[0:4096]
_baseurl_cache[response] = html.get_base_url(text, response.url,
response.encoding)
return _baseurl_cache[response]
@@ -37,7 +37,7 @@ _metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
"""Parse the http-equiv refrsh parameter from the given response"""
if response not in _metaref_cache:
- text = response.body_as_unicode()[0:4096]
+ text = response.text[0:4096]
text = _noscript_re.sub(u'', text)
text = _script_re.sub(u'', text)
_metaref_cache[response] = html.get_meta_refresh(text, response.url,
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 9f2c02bff..baf6ef1bf 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -55,12 +55,11 @@ class TestSpider(Spider):
def parse_item(self, response):
item = self.item_cls()
- body = response.body_as_unicode()
- m = self.name_re.search(body)
+ m = self.name_re.search(response.text)
if m:
item['name'] = m.group(1)
item['url'] = response.url
- m = self.price_re.search(body)
+ m = self.price_re.search(response.text)
if m:
item['price'] = m.group(1)
return item
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 710a5b29d..c7f36687a 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -107,9 +107,11 @@ class BaseResponseTest(unittest.TestCase):
body_bytes = body
assert isinstance(response.body, bytes)
+ assert isinstance(response.text, six.text_type)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
self.assertEqual(response.body_as_unicode(), body_unicode)
+ self.assertEqual(response.text, body_unicode)
def _assert_response_encoding(self, response, encoding):
self.assertEqual(response.encoding, resolve_encoding(encoding))
@@ -171,6 +173,10 @@ class TextResponseTest(BaseResponseTest):
self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
self.assertEqual(r1.body_as_unicode(), unicode_string)
+ # check response.text
+ self.assertTrue(isinstance(r1.text, six.text_type))
+ self.assertEqual(r1.text, unicode_string)
+
def test_encoding(self):
r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body=b"\xc2\xa3")
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
@@ -219,12 +225,12 @@ class TextResponseTest(BaseResponseTest):
headers={"Content-type": ["text/html; charset=utf-8"]},
body=b"\xef\xbb\xbfWORD\xe3\xab")
self.assertEqual(r6.encoding, 'utf-8')
- self.assertEqual(r6.body_as_unicode(), u'WORD\ufffd\ufffd')
+ self.assertEqual(r6.text, u'WORD\ufffd\ufffd')
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
- # response.body_as_unicode() in indistint order doesn't affect final
+ # response.text in indistint order doesn't affect final
# values for encoding and decoded body.
url = 'http://example.com'
body = b"\xef\xbb\xbfWORD"
@@ -233,9 +239,9 @@ class TextResponseTest(BaseResponseTest):
# Test response without content-type and BOM encoding
response = self.response_class(url, body=body)
self.assertEqual(response.encoding, 'utf-8')
- self.assertEqual(response.body_as_unicode(), u'WORD')
+ self.assertEqual(response.text, u'WORD')
response = self.response_class(url, body=body)
- self.assertEqual(response.body_as_unicode(), u'WORD')
+ self.assertEqual(response.text, u'WORD')
self.assertEqual(response.encoding, 'utf-8')
# Body caching sideeffect isn't triggered when encoding is declared in
@@ -243,9 +249,9 @@ class TextResponseTest(BaseResponseTest):
# body
response = self.response_class(url, headers=headers, body=body)
self.assertEqual(response.encoding, 'utf-8')
- self.assertEqual(response.body_as_unicode(), u'WORD')
+ self.assertEqual(response.text, u'WORD')
response = self.response_class(url, headers=headers, body=body)
- self.assertEqual(response.body_as_unicode(), u'WORD')
+ self.assertEqual(response.text, u'WORD')
self.assertEqual(response.encoding, 'utf-8')
def test_replace_wrong_encoding(self):
@@ -253,18 +259,18 @@ class TextResponseTest(BaseResponseTest):
r = self.response_class("http://www.example.com", encoding='utf-8', body=b'PREFIX\xe3\xabSUFFIX')
# XXX: Policy for replacing invalid chars may suffer minor variations
# but it should always contain the unicode replacement char (u'\ufffd')
- assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
- assert u'PREFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
- assert u'SUFFIX' in r.body_as_unicode(), repr(r.body_as_unicode())
+ assert u'\ufffd' in r.text, repr(r.text)
+ assert u'PREFIX' in r.text, repr(r.text)
+ assert u'SUFFIX' in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
r = self.response_class("http://example.com", encoding='utf-8', \
body=b'\xf0<span>value</span>')
- assert u'<span>value</span>' in r.body_as_unicode(), repr(r.body_as_unicode())
+ assert u'<span>value</span>' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
- #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX')
- #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode())
+ #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
+ #assert u'\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"