Scrapy, a fast high-level web crawling & scraping framework for Python.
Skip invalid links when passing SelectorLists to Response.follow_all
Eugenio Lacuesta2a9f5a0aefae
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 6acf1026f..e3646b2d5 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -7,10 +7,10 @@ See documentation in docs/topics/request-response.rstfrom contextlib import suppressfrom typing import Generator+from urllib.parse import urljoinimport parselimport six-from six.moves.urllib.parse import urljoinfrom w3lib.encoding import (html_body_declared_encoding, html_to_unicode,http_content_type_encoding, resolve_encoding)from w3lib.html import strip_html5_whitespace@@ -183,22 +183,25 @@ class TextResponse(Response):In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extractionwithin the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).- Note that when using the ``css`` or ``xpath`` parameters, this method will not produce- requests for selectors from which links cannot be obtained (for instance, anchor tags- without an ``href`` attribute)+ Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or+ using the ``css`` or ``xpath`` parameters, this method will not produce requests for+ selectors from which links cannot be obtained (for instance, anchor tags without an+ ``href`` attribute)"""arg_count = len(list(filter(None, (urls, css, xpath))))if arg_count != 1:raise ValueError('Please supply exactly one of the following arguments: urls, css, xpath')if not urls:if css:- selector_list = self.css(css)+ urls = self.css(css)if xpath:- selector_list = self.xpath(xpath)+ urls = self.xpath(xpath)+ if isinstance(urls, parsel.SelectorList):+ selectors = urlsurls = []- for selector in selector_list:+ for sel in selectors:with suppress(_InvalidSelector):- urls.append(_url_from_selector(selector))+ urls.append(_url_from_selector(sel))return super(TextResponse, self).follow_all(urls=urls,callback=callback,diff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 36ccdfa1f..ce13650ce 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -579,8 +579,10 @@ class TextResponseTest(BaseResponseTest):'http://example.com/page/4/',]response = self._links_response_no_href()- extracted = [r.url for r in response.follow_all(css='.pagination a')]- self.assertEqual(expected, extracted)+ extracted1 = [r.url for r in response.follow_all(css='.pagination a')]+ self.assertEqual(expected, extracted1)+ extracted2 = [r.url for r in response.follow_all(response.css('.pagination a'))]+ self.assertEqual(expected, extracted2)def test_follow_all_xpath(self):expected = [@@ -598,8 +600,10 @@ class TextResponseTest(BaseResponseTest):'http://example.com/page/4/',]response = self._links_response_no_href()- extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]- self.assertEqual(expected, extracted)+ extracted1 = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]+ self.assertEqual(expected, extracted1)+ extracted2 = [r.url for r in response.follow_all(response.xpath('//div[@id="pagination"]/a'))]+ self.assertEqual(expected, extracted2)def test_follow_all_too_many_arguments(self):response = self._links_response()