Scrapy, a fast high-level web crawling & scraping framework for Python.
Added: Now supporting <link> tags in Response.follow
Pengyu CHENb33e0d5a54a9
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 6415e191a..74a042f2c 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -135,7 +135,7 @@ class TextResponse(Response):* an attribute Selector (not SelectorList) - e.g.``response.css('a::attr(href)')[0]`` or``response.xpath('//img/@src')[0]``.- * a Selector for ``<a>`` element, e.g.+ * a Selector for ``<a>`` or ``<link>`` element, e.g.``response.css('a.my_link')[0]``.See :ref:`response-follow-example` for usage examples.@@ -165,10 +165,11 @@ def _url_from_selector(sel):return strip_html5_whitespace(sel.root)if not hasattr(sel.root, 'tag'):raise ValueError("Unsupported selector: %s" % sel)- if sel.root.tag != 'a':- raise ValueError("Only <a> elements are supported; got <%s>" %+ if sel.root.tag not in ('a', 'link'):+ raise ValueError("Only <a> and <link> elements are supported; got <%s>" %sel.root.tag)href = sel.root.get('href')if href is None:- raise ValueError("<a> element has no href attribute: %s" % sel)+ raise ValueError("<%s> element has no href attribute: %s" %+ (sel.root.tag, sel))return strip_html5_whitespace(href)diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 779f5a71c..a36ec3af6 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -162,7 +162,6 @@ class BaseResponseTest(unittest.TestCase):def test_follow_whitespace_link(self):self._assert_followed_url(Link('http://example.com/foo '),'http://example.com/foo%20')-def _assert_followed_url(self, follow_obj, target_url, response=None):if response is None:response = self._links_response()@@ -402,6 +401,13 @@ class TextResponseTest(BaseResponseTest):for sel, url in zip(sellist, urls):self._assert_followed_url(sel, url, response=resp)+ # select <link> elements+ self._assert_followed_url(+ Selector(text='<link href="foo"></link>').css('link')[0],+ 'http://example.com/foo',+ response=resp+ )+# href attributes should workfor sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:for sel, url in zip(sellist, urls):