Scrapy, a fast high-level web crawling & scraping framework for Python.
Response.follow_all: Specific exception for invalid selectors
Eugenio Lacuestae6c5292a7c43
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 74017b5aa..5110b4bd4 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -5,17 +5,18 @@ discovering (through HTTP headers) to base Response class.See documentation in docs/topics/request-response.rst"""-import six-from six.moves.urllib.parse import urljoin+from contextlib import suppressimport parsel-from w3lib.encoding import html_to_unicode, resolve_encoding, \- html_body_declared_encoding, http_content_type_encoding+import six+from six.moves.urllib.parse import urljoin+from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,+ http_content_type_encoding, resolve_encoding)from w3lib.html import strip_html5_whitespacefrom scrapy.http.response import Response-from scrapy.utils.response import get_base_urlfrom scrapy.utils.python import memoizemethod_noargs, to_native_str+from scrapy.utils.response import get_base_urlclass TextResponse(Response):@@ -197,10 +198,8 @@ class TextResponse(Response):selector_list = self.xpath(xpath)urls = []for selector in selector_list:- try:+ with suppress(_InvalidSelector):urls.append(_url_from_selector(selector))- except ValueError:- passreturn super(TextResponse, self).follow_all(urls=urls,callback=callback,@@ -217,18 +216,24 @@ class TextResponse(Response):)+class _InvalidSelector(ValueError):+ """+ Raised when a URL cannot be obtained from a Selector+ """++def _url_from_selector(sel):# type: (parsel.Selector) -> strif isinstance(sel.root, six.string_types):# e.g. ::attr(href) resultreturn strip_html5_whitespace(sel.root)if not hasattr(sel.root, 'tag'):- raise ValueError("Unsupported selector: %s" % sel)+ raise _InvalidSelector("Unsupported selector: %s" % sel)if sel.root.tag not in ('a', 'link'):- raise ValueError("Only <a> and <link> elements are supported; got <%s>" %- sel.root.tag)+ raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %+ sel.root.tag)href = sel.root.get('href')if href is None:- raise ValueError("<%s> element has no href attribute: %s" %- (sel.root.tag, sel))+ raise _InvalidSelector("<%s> element has no href attribute: %s" %+ (sel.root.tag, sel))return strip_html5_whitespace(href)