Scrapy, a fast high-level web crawling & scraping framework for Python.
Response.follow_all
Eugenio Lacuesta5f168cd459cf
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 727c67482..9fe3c7518 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -701,6 +701,8 @@ Response objects.. automethod:: Response.follow+ .. automethod:: Response.follow_all+.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin@@ -790,6 +792,8 @@ TextResponse objects.. automethod:: TextResponse.follow+ .. automethod:: TextResponse.follow_all+.. method:: TextResponse.body_as_unicode()The same as :attr:`text`, but available as a method. This method isdiff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.pyindex b0a526b72..96359705e 100644--- a/scrapy/http/response/__init__.py+++ b/scrapy/http/response/__init__.py@@ -113,8 +113,8 @@ class Response(object_ref):It accepts the same arguments as ``Request.__init__`` method,but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,not only an absolute URL.-- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`++ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`method which supports selectors in addition to absolute/relative URLsand Link objects."""@@ -123,14 +123,51 @@ class Response(object_ref):elif url is None:raise ValueError("url can't be None")url = self.urljoin(url)- return Request(url, callback,- method=method,- headers=headers,- body=body,- cookies=cookies,- meta=meta,- encoding=encoding,- priority=priority,- dont_filter=dont_filter,- errback=errback,- cb_kwargs=cb_kwargs)+ return Request(+ url=url,+ callback=callback,+ method=method,+ headers=headers,+ body=body,+ cookies=cookies,+ meta=meta,+ encoding=encoding,+ priority=priority,+ dont_filter=dont_filter,+ errback=errback,+ cb_kwargs=cb_kwargs,+ )++ def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,+ cookies=None, meta=None, encoding='utf-8', priority=0,+ dont_filter=False, errback=None, cb_kwargs=None):+ # type: (...) -> Generator[Request, None, None]+ """+ Return an iterable of :class:`~.Request` instance to follow all links+ in ``urls``. It accepts the same arguments as ``Request.__init__`` method,+ but elements of ``urls`` can be relative URLs or ``scrapy.link.Link`` objects+ not only absolute URLs.++ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`+ method which supports selectors in addition to absolute/relative URLs+ and Link objects.+ """+ if not hasattr(urls, '__iter__'):+ raise TypeError("'urls' argument must be an iterable")+ return (+ self.follow(+ url=url,+ callback=callback,+ method=method,+ headers=headers,+ body=body,+ cookies=cookies,+ meta=meta,+ encoding=encoding,+ priority=priority,+ dont_filter=dont_filter,+ errback=errback,+ cb_kwargs=cb_kwargs,+ )+ for url in urls+ )diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 339913d4e..81e3f9b28 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -13,7 +13,6 @@ from w3lib.encoding import html_to_unicode, resolve_encoding, \html_body_declared_encoding, http_content_type_encodingfrom w3lib.html import strip_html5_whitespace-from scrapy.http.request import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.response import get_base_urlfrom scrapy.utils.python import memoizemethod_noargs, to_native_str@@ -44,7 +43,7 @@ class TextResponse(Response):if isinstance(body, six.text_type):if self._encoding is None:raise TypeError('Cannot convert unicode body - %s has no encoding' %- type(self).__name__)+ type(self).__name__)self._body = body.encode(self._encoding)else:super(TextResponse, self)._set_body(body)@@ -90,8 +89,8 @@ class TextResponse(Response):if self._cached_benc is None:content_type = to_native_str(self.headers.get(b'Content-Type', b''))benc, ubody = html_to_unicode(content_type, self.body,- auto_detect_fun=self._auto_detect_fun,- default_encoding=self._DEFAULT_ENCODING)+ auto_detect_fun=self._auto_detect_fun,+ default_encoding=self._DEFAULT_ENCODING)self._cached_benc = bencself._cached_ubody = ubodyreturn self._cached_benc@@ -129,7 +128,7 @@ class TextResponse(Response):Return a :class:`~.Request` instance to follow a link ``url``.It accepts the same arguments as ``Request.__init__`` method,but ``url`` can be not only an absolute URL, but also-+* a relative URL;* a scrapy.link.Link object (e.g. a link extractor result);* an attribute Selector (not SelectorList) - e.g.@@ -137,7 +136,7 @@ class TextResponse(Response):``response.xpath('//img/@src')[0]``.* a Selector for ``<a>`` or ``<link>`` element, e.g.``response.css('a.my_link')[0]``.-+See :ref:`response-follow-example` for usage examples."""if isinstance(url, parsel.Selector):@@ -145,7 +144,9 @@ class TextResponse(Response):elif isinstance(url, parsel.SelectorList):raise ValueError("SelectorList is not supported")encoding = self.encoding if encoding is None else encoding- return super(TextResponse, self).follow(url, callback,+ return super(TextResponse, self).follow(+ url=url,+ callback=callback,method=method,headers=headers,body=body,@@ -158,6 +159,52 @@ class TextResponse(Response):cb_kwargs=cb_kwargs,)+ def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,+ cookies=None, meta=None, encoding=None, priority=0,+ dont_filter=False, errback=None, cb_kwargs=None,+ css=None, xpath=None):+ # type: (...) -> Generator[Request, None, None]+ """+ A generator that produces :class:`~.Request` instances to follow all+ links in ``urls``. It accepts the same arguments as the :class:`~.Request`+ initializer, except that each ``urls`` element does not need to be an absolute+ URL, it can be any of the following:++ * a relative URL;+ * a scrapy.link.Link object (e.g. a link extractor result);+ * an attribute Selector (not SelectorList) - e.g.+ ``response.css('a::attr(href)')[0]`` or+ ``response.xpath('//img/@src')[0]``.+ * a Selector for ``<a>`` or ``<link>`` element, e.g.+ ``response.css('a.my_link')[0]``.++ In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction+ within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` are accepted).+ """+ if len(list(filter(None, (urls, css, xpath)))) > 1:+ raise ValueError('Please supply only one of the following arguments: {urls, css, xpath}')+ if css:+ urls = self.css(css)+ elif xpath:+ urls = self.xpath(xpath)+ return (+ self.follow(+ url=url,+ callback=callback,+ method=method,+ headers=headers,+ body=body,+ cookies=cookies,+ meta=meta,+ encoding=encoding,+ priority=priority,+ dont_filter=dont_filter,+ errback=errback,+ cb_kwargs=cb_kwargs,+ )+ for url in urls+ )+def _url_from_selector(sel):# type: (parsel.Selector) -> strdiff --git a/tests/test_http_response.py b/tests/test_http_response.pyindex cd5c3486e..134856c78 100644--- a/tests/test_http_response.py+++ b/tests/test_http_response.py@@ -143,6 +143,8 @@ class BaseResponseTest(unittest.TestCase):r.css('body')r.xpath('//body')+ # Response.follow+def test_follow_url_absolute(self):self._assert_followed_url('http://foo.example.com','http://foo.example.com')@@ -166,6 +168,64 @@ class BaseResponseTest(unittest.TestCase):def test_follow_whitespace_link(self):self._assert_followed_url(Link('http://example.com/foo '),'http://example.com/foo%20')++ # Response.follow_all++ def test_follow_all_absolute(self):+ url_list = ['http://example.org', 'http://www.example.org',+ 'http://example.com', 'http://www.example.com']+ self._assert_followed_all_urls(url_list, url_list)++ def test_follow_all_relative(self):+ relative = ['foo', 'bar', 'foo/bar', 'bar/foo']+ absolute = [+ 'http://example.com/foo',+ 'http://example.com/bar',+ 'http://example.com/foo/bar',+ 'http://example.com/bar/foo',+ ]+ self._assert_followed_all_urls(relative, absolute)++ def test_follow_all_links(self):+ absolute = [+ 'http://example.com/foo',+ 'http://example.com/bar',+ 'http://example.com/foo/bar',+ 'http://example.com/bar/foo',+ ]+ links = map(Link, absolute)+ self._assert_followed_all_urls(links, absolute)++ def test_follow_all_invalid(self):+ r = self.response_class("http://example.com")+ with self.assertRaises(TypeError):+ list(r.follow_all(None))+ with self.assertRaises(TypeError):+ list(r.follow_all(12345))+ with self.assertRaises(ValueError):+ list(r.follow_all([None]))++ def test_follow_all_whitespace(self):+ relative = ['foo ', 'bar ', 'foo/bar ', 'bar/foo ']+ absolute = [+ 'http://example.com/foo%20',+ 'http://example.com/bar%20',+ 'http://example.com/foo/bar%20',+ 'http://example.com/bar/foo%20',+ ]+ self._assert_followed_all_urls(relative, absolute)++ def test_follow_all_whitespace_links(self):+ absolute = [+ 'http://example.com/foo ',+ 'http://example.com/bar ',+ 'http://example.com/foo/bar ',+ 'http://example.com/bar/foo ',+ ]+ links = map(Link, absolute)+ expected = [u.replace(' ', '%20') for u in absolute]+ self._assert_followed_all_urls(links, expected)+def _assert_followed_url(self, follow_obj, target_url, response=None):if response is None:response = self._links_response()@@ -173,6 +233,14 @@ class BaseResponseTest(unittest.TestCase):self.assertEqual(req.url, target_url)return req+ def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):+ if response is None:+ response = self._links_response()+ followed = response.follow_all(follow_obj)+ for req, target in zip(followed, target_urls):+ self.assertEqual(req.url, target)+ yield req+def _links_response(self):body = get_testdata('link_extractor', 'sgml_linkextractor.html')resp = self.response_class('http://example.com/index', body=body)@@ -483,6 +551,29 @@ class TextResponseTest(BaseResponseTest):)self.assertEqual(req.encoding, 'cp1251')+ def test_follow_all_css(self):+ expected = [+ 'http://example.com/sample3.html',+ 'http://example.com/innertag.html',+ ]+ response = self._links_response()+ extracted = [r.url for r in response.follow_all(css='a[href*="example.com"]')]+ self.assertEqual(expected, extracted)++ def test_follow_all_xpath(self):+ expected = [+ 'http://example.com/sample3.html',+ 'http://example.com/innertag.html',+ ]+ response = self._links_response()+ extracted = response.follow_all(xpath='//a[contains(@href, "example.com")]')+ self.assertEqual(expected, [r.url for r in extracted])++ def test_follow_all_exception(self):+ response = self._links_response()+ with self.assertRaises(ValueError):+ response.follow_all(css='a[href*="example.com"]', xpath='//a[contains(@href, "example.com")]')+class HtmlResponseTest(TextResponseTest):