Scrapy, a fast high-level web crawling & scraping framework for Python.
TextResponse.follow_all: skip invalid links
Eugenio Lacuestae1fa1fd8ad62
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 81e3f9b28..ccacce550 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -180,13 +180,27 @@ class TextResponse(Response):In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extractionwithin the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).++ Note that when using the ``css`` or ``xpath`` parameters, this method will not produce+ requests for selectors from which links cannot be obtained (for instance, anchor tags+ without ``href`` attribute)"""- if len(list(filter(None, (urls, css, xpath)))) > 1:- raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')- if css:- urls = self.css(css)- elif xpath:- urls = self.xpath(xpath)+ arg_count = len(list(filter(None, (urls, css, xpath))))+ if arg_count != 1:+ raise ValueError('Please supply exactly one of the following arguments: {urls, css, xpath}')+ if not urls:+ urls = []+ if css:+ selector_method = getattr(self, 'css')+ expression = css+ elif xpath:+ selector_method = getattr(self, 'xpath')+ expression = xpath+ for selector in selector_method(expression):+ try:+ urls.append(_url_from_selector(selector))+ except ValueError:+ passreturn (self.follow(url=url,diff --git a/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.htmlnew file mode 100644index 000000000..0b01cede8--- /dev/null+++ b/tests/sample_data/link_extractor/sgml_linkextractor_no_href.html@@ -0,0 +1,25 @@+<html>+ <head>+ <base href='http://example.com' />+ <title>Sample page with anchor tags containing no href attribute, to test the TextResponse.follow_all method</title>+ </head>++<body>+ <div class="quote">+ <span class="text">“The world as we have created it is a process of our+ thinking. It cannot be changed without changing our thinking.”</span>+ <span>+ by <small class="author">Albert Einstein</small>+ <a href="/author/Albert-Einstein">(about)</a>+ </span>+ <div id="pagination" class="pagination">+ Tags:+ <a href="/page/1/">Page 1</a>+ <a>Current</a>+ <a href="/page/3/">Page 3</a>+ <a href="/page/4/">Page 4</a>+ </div>+ </div>+</body>++</html>\ No newline at end of filediff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex 134856c78..6d3c5cb9d 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -198,12 +198,20 @@ class BaseResponseTest(unittest.TestCase):def test_follow_all_invalid(self):r = self.response_class("http://example.com")- with self.assertRaises(TypeError):- list(r.follow_all(None))- with self.assertRaises(TypeError):- list(r.follow_all(12345))- with self.assertRaises(ValueError):- list(r.follow_all([None]))+ if self.response_class == Response:+ with self.assertRaises(TypeError):+ list(r.follow_all(urls=None))+ with self.assertRaises(TypeError):+ list(r.follow_all(urls=12345))+ with self.assertRaises(ValueError):+ list(r.follow_all(urls=[None]))+ else:+ with self.assertRaises(ValueError):+ list(r.follow_all(urls=None))+ with self.assertRaises(TypeError):+ list(r.follow_all(urls=12345))+ with self.assertRaises(ValueError):+ list(r.follow_all(urls=[None]))def test_follow_all_whitespace(self):relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']@@ -246,6 +254,11 @@ class BaseResponseTest(unittest.TestCase):resp = self.response_class('http://example.com/index', body=body)return resp+ def _links_response_no_href(self):+ body = get_testdata('link_extractor', 'sgml_linkextractor_no_href.html')+ resp = self.response_class('http://example.com/index', body=body)+ return resp+class TextResponseTest(BaseResponseTest):@@ -560,6 +573,16 @@ class TextResponseTest(BaseResponseTest):extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]self.assertEqual(expected, extracted)+ def test_follow_all_css_skip_invalid(self):+ expected = [+ 'http://example.com/page/1/',+ 'http://example.com/page/3/',+ 'http://example.com/page/4/',+ ]+ response = self._links_response_no_href()+ extracted = [r.url for r in response.follow_all(css='.pagination a')]+ self.assertEqual(expected, extracted)+def test_follow_all_xpath(self):expected = ['http://example.com/sample3.html',@@ -569,7 +592,17 @@ class TextResponseTest(BaseResponseTest):extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')self.assertEqual(expected, [r.url for r in extracted])- def test_follow_all_exception(self):+ def test_follow_all_xpath_skip_invalid(self):+ expected = [+ 'http://example.com/page/1/',+ 'http://example.com/page/3/',+ 'http://example.com/page/4/',+ ]+ response = self._links_response_no_href()+ extracted = [r.url for r in response.follow_all(xpath='//div[@id="pagination"]/a')]+ self.assertEqual(expected, extracted)++ def test_follow_all_too_many_arguments(self):response = self._links_response()with self.assertRaises(ValueError):response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')