Scrapy, a fast high-level web crawling & scraping framework for Python.
handle whitespace in response.follow; add tests
Mikhail Korobov608c3f0c452b
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 5bfd2debb..6eacfbd35 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -11,6 +11,7 @@ from six.moves.urllib.parse import urljoinimport parselfrom w3lib.encoding import html_to_unicode, resolve_encoding, \html_body_declared_encoding, http_content_type_encoding+from w3lib.html import strip_html5_whitespacefrom scrapy.link import Linkfrom scrapy.http.request import Request@@ -142,7 +143,7 @@ class TextResponse(Response):if isinstance(url, Link):url = url.urlelif isinstance(url, parsel.Selector):- url = _url_from_selector(url).strip()+ url = _url_from_selector(url)elif isinstance(url, parsel.SelectorList):raise ValueError("SelectorList is not supported")@@ -164,7 +165,7 @@ def _url_from_selector(sel):# type: (parsel.Selector) -> strif isinstance(sel.root, six.string_types):# e.g. ::attr(href) result- return sel.root+ return strip_html5_whitespace(sel.root)if not hasattr(sel.root, 'tag'):raise ValueError("Unsupported selector: %s" % sel)if sel.root.tag != 'a':@@ -173,4 +174,4 @@ def _url_from_selector(sel):href = sel.root.get('href')if href is None:raise ValueError("<a> element has no href attribute: %s" % sel)- return href+ return strip_html5_whitespace(href)diff --git a/tests/__init__.py b/tests/__init__.pyindex d940f28ea..c2e4fd2bf 100644--- a/tests/__init__.py+++ b/tests/__init__.py@@ -26,9 +26,12 @@ try:except ImportError:import mock-tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')+tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),+ 'sample_data')+def get_testdata(*paths):"""Return test data"""path = os.path.join(tests_datadir, *paths)- return open(path, 'rb').read()+ with open(path, 'rb') as f:+ return f.read()diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 9df3bf6e7..2a9baf5ed 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -1,3 +1,4 @@+# -*- coding: utf-8 -*-import unittestimport six@@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,from scrapy.selector import Selectorfrom scrapy.utils.python import to_native_strfrom scrapy.exceptions import NotSupported+from scrapy.link import Link+from tests import get_testdataclass BaseResponseTest(unittest.TestCase):@@ -356,6 +359,11 @@ class HtmlResponseTest(TextResponseTest):response_class = HtmlResponse+ def _links_response(self):+ body = get_testdata('link_extractor', 'sgml_linkextractor.html')+ resp = self.response_class('http://example.com/index', body=body)+ return resp+def test_html_encoding(self):body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">@@ -388,6 +396,103 @@ class HtmlResponseTest(TextResponseTest):r1 = self.response_class("http://www.example.com", body=body)self._assert_response_values(r1, 'gb2312', body)+ def assert_followed_url(self, follow_obj, target_url, response=None):+ if response is None:+ response = self._links_response()+ req = response.follow(follow_obj)+ self.assertEqual(req.url, target_url)+ return req++ def test_follow_url_absolute(self):+ self.assert_followed_url('http://foo.example.com',+ 'http://foo.example.com')++ def test_follow_url_relative(self):+ self.assert_followed_url('foo',+ 'http://example.com/foo')++ def test_follow_link(self):+ self.assert_followed_url(Link('http://example.com/foo'),+ 'http://example.com/foo')++ def test_follow_selector(self):+ resp = self._links_response()+ urls = [+ 'http://example.com/sample2.html',+ 'http://example.com/sample3.html',+ 'http://example.com/sample3.html',+ 'http://www.google.com/something',+ 'http://example.com/innertag.html'+ ]++ # select <a> elements+ for sellist in [resp.css('a'), resp.xpath('//a')]:+ for sel, url in zip(sellist, urls):+ self.assert_followed_url(sel, url, response=resp)++ # href attributes should work+ for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:+ for sel, url in zip(sellist, urls):+ self.assert_followed_url(sel, url, response=resp)++ # non-a elements are not supported+ self.assertRaises(ValueError, resp.follow, resp.css('div')[0])++ def test_follow_selector_list(self):+ resp = self._links_response()+ self.assertRaisesRegex(ValueError, 'SelectorList',+ resp.follow, resp.css('a'))++ def test_follow_selector_attribute(self):+ resp = self._links_response()+ for src in resp.css('img::attr(src)'):+ self.assert_followed_url(src, 'http://example.com/sample2.jpg')++ def test_follow_whitespace_url(self):+ self.assert_followed_url('foo ',+ 'http://example.com/foo%20')++ def test_follow_whitespace_link(self):+ self.assert_followed_url(Link('http://example.com/foo '),+ 'http://example.com/foo%20')++ def test_follow_whitespace_selector(self):+ resp = self.response_class(+ 'http://example.com',+ body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''+ )+ self.assert_followed_url(resp.css('a')[0],+ 'http://example.com/foo',+ response=resp)+ self.assert_followed_url(resp.css('a::attr(href)')[0],+ 'http://example.com/foo',+ response=resp)++ def test_follow_encoding(self):+ resp1 = self.response_class(+ 'http://example.com',+ encoding='utf8',+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')+ )+ req = self.assert_followed_url(+ resp1.css('a')[0],+ 'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',+ response=resp1,+ )+ self.assertEqual(req.encoding, 'utf8')++ resp2 = self.response_class(+ 'http://example.com',+ encoding='cp1251',+ body='<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')+ )+ req = self.assert_followed_url(+ resp2.css('a')[0],+ 'http://example.com/foo?%EF%F0%E8%E2%E5%F2',+ response=resp2,+ )+ self.assertEqual(req.encoding, 'cp1251')+class XmlResponseTest(TextResponseTest):