Scrapy, a fast high-level web crawling & scraping framework for Python.
initial response.follow implementation
Mikhail Korobov877057fac0d4
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rstindex 7195017ff..1da1a4059 100644--- a/docs/intro/overview.rst+++ b/docs/intro/overview.rst@@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::next_page = response.css('li.next a::attr("href")').extract_first()if next_page is not None:- next_page = response.urljoin(next_page)- yield scrapy.Request(next_page, callback=self.parse)+ yield response.follow(next_page, self.parse)Put this in a text file, name it to something like ``quotes_spider.py``diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rstindex 3dc5ad2ed..d47bf69e5 100644--- a/docs/intro/tutorial.rst+++ b/docs/intro/tutorial.rst@@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next puntil it doesn't find one -- handy for crawling blogs, forums and other sites withpagination.+As a shortcut for creating Request objects you can use+:meth:`response.follow <scrapy.http.TextResponse.follow>` method::++ import scrapy+++ class QuotesSpider(scrapy.Spider):+ name = "quotes"+ start_urls = [+ 'http://quotes.toscrape.com/page/1/',+ ]++ def parse(self, response):+ for quote in response.css('div.quote'):+ yield {+ 'text': quote.css('span.text::text').extract_first(),+ 'author': quote.css('span small::text').extract_first(),+ 'tags': quote.css('div.tags a.tag::text').extract(),+ }++ for href in response.css('li.next a::attr(href)'):+ yield response.follow(href, callback=self.parse)++Unlike scrapy.Request, ``response.follow`` supports+relative URLs directly; you can also pass a selector to it instead of+a string. Note that ``response.follow`` just returns a Request instance;+you still have to yield this Request.+More examples and patterns--------------------------Here is another spider that illustrates callbacks and following links,this time for scraping author information::-import scrapy@@ -568,15 +595,12 @@ this time for scraping author information::def parse(self, response):# follow links to author pages- for href in response.css('.author + a::attr(href)').extract():- yield scrapy.Request(response.urljoin(href),- callback=self.parse_author)+ for href in response.css('.author + a::attr(href)'):+ yield response.follow(href, self.parse_author)# follow pagination links- next_page = response.css('li.next a::attr(href)').extract_first()- if next_page is not None:- next_page = response.urljoin(next_page)- yield scrapy.Request(next_page, callback=self.parse)+ for href in response.css('li.next a::attr(href)'):+ yield response.follow(href, self.parse)def parse_author(self, response):def extract_with_css(query):@@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to theauthors pages calling the ``parse_author`` callback for each of them, and alsothe pagination links with the ``parse`` callback as we saw before.+Here we're passing callbacks to ``response.follow`` as positional arguments+to make the code shorter; it also works for ``scrapy.Request``.+The ``parse_author`` callback defines a helper function to extract and cleanup thedata from a CSS query and yields the Python dict with the author data.@@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument::next_page = response.css('li.next a::attr(href)').extract_first()if next_page is not None:- next_page = response.urljoin(next_page)- yield scrapy.Request(next_page, self.parse)+ yield response.follow(next_page, self.parse)If you pass the ``tag=humor`` argument to this spider, you'll notice that itdiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 1fdd26043..71050fddd 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -683,6 +683,10 @@ TextResponse objectsresponse.css('p')+ .. method:: TextResponse.follow(url, ...)++ Return a scrapy.Request instance to follow a link ``url``.+.. method:: TextResponse.body_as_unicode()The same as :attr:`text`, but available as a method. This method isdiff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.pyindex 5a6507aa8..1718b1f3b 100644--- a/scrapy/http/response/text.py+++ b/scrapy/http/response/text.py@@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rstimport sixfrom six.moves.urllib.parse import urljoin+import parselfrom w3lib.encoding import html_to_unicode, resolve_encoding, \html_body_declared_encoding, http_content_type_encoding++from scrapy.link import Link+from scrapy.http.request import Requestfrom scrapy.http.response import Responsefrom scrapy.utils.response import get_base_urlfrom scrapy.utils.python import memoizemethod_noargs, to_native_str@@ -116,3 +120,58 @@ class TextResponse(Response):def css(self, query):return self.selector.css(query)++ def follow(self, url, callback=None, method='GET', headers=None, body=None,+ cookies=None, meta=None, encoding=None, priority=0,+ dont_filter=False, errback=None):+ # type: (...) -> Request+ """+ Return a scrapy.Request instance to follow a link ``url``.++ ``url`` can be:++ * absolute URL;+ * relative URL;+ * scrapy.link.Link object (e.g. a link extractor result);+ * attribute Selector (not SelectorList) - e.g.+ ``response.css('a::attr(href)')[0]`` or+ ``response.xpath('//img/@src')[0]``.+ * a Selector for ``<a>`` element, e.g.+ ``response.css('a.my_link')[0]``.+ """+ if isinstance(url, Link):+ url = url.url+ elif isinstance(url, parsel.Selector):+ url = _url_from_selector(url)+ elif isinstance(url, parsel.SelectorList):+ raise ValueError("Please pass either string")+++ encoding = self.encoding if encoding is None else encoding+ url = self.urljoin(url)+ return Request(url, callback,+ method=method,+ headers=headers,+ body=body,+ cookies=cookies,+ meta=meta,+ encoding=encoding,+ priority=priority,+ dont_filter=dont_filter,+ errback=errback)+++def _url_from_selector(sel):+ # type: (parsel.Selector) -> str+ if isinstance(sel.root, six.string_types):+ # e.g. ::attr(href) result+ return sel.root+ if not hasattr(sel.root, 'tag'):+ raise ValueError("Unsupported selector: %s" % sel)+ if sel.root.tag != 'a':+ raise ValueError("Only <a> elements are supported; got <%s>" %+ sel.root.tag)+ href = sel.root.get('href')+ if href is None:+ raise ValueError("<a> element has no href attribute: %s" % sel)+ return href