Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

initial response.follow implementation

Mikhail Korobov877057fac0d4
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 7195017ff..1da1a4059 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::
next_page = response.css('li.next a::attr("href")').extract_first()
if next_page is not None:
- next_page = response.urljoin(next_page)
- yield scrapy.Request(next_page, callback=self.parse)
+ yield response.follow(next_page, self.parse)
Put this in a text file, name it to something like ``quotes_spider.py``
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 3dc5ad2ed..d47bf69e5 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -551,13 +551,40 @@ In our example, it creates a sort of loop, following all the links to the next p
until it doesn't find one -- handy for crawling blogs, forums and other sites with
pagination.
+As a shortcut for creating Request objects you can use
+:meth:`response.follow <scrapy.http.TextResponse.follow>` method::
+
+ import scrapy
+
+
+ class QuotesSpider(scrapy.Spider):
+ name = "quotes"
+ start_urls = [
+ 'http://quotes.toscrape.com/page/1/',
+ ]
+
+ def parse(self, response):
+ for quote in response.css('div.quote'):
+ yield {
+ 'text': quote.css('span.text::text').extract_first(),
+ 'author': quote.css('span small::text').extract_first(),
+ 'tags': quote.css('div.tags a.tag::text').extract(),
+ }
+
+ for href in response.css('li.next a::attr(href)'):
+ yield response.follow(href, callback=self.parse)
+
+Unlike scrapy.Request, ``response.follow`` supports
+relative URLs directly; you can also pass a selector to it instead of
+a string. Note that ``response.follow`` just returns a Request instance;
+you still have to yield this Request.
+
More examples and patterns
--------------------------
Here is another spider that illustrates callbacks and following links,
this time for scraping author information::
-
import scrapy
@@ -568,15 +595,12 @@ this time for scraping author information::
def parse(self, response):
# follow links to author pages
- for href in response.css('.author + a::attr(href)').extract():
- yield scrapy.Request(response.urljoin(href),
- callback=self.parse_author)
+ for href in response.css('.author + a::attr(href)'):
+ yield response.follow(href, self.parse_author)
# follow pagination links
- next_page = response.css('li.next a::attr(href)').extract_first()
- if next_page is not None:
- next_page = response.urljoin(next_page)
- yield scrapy.Request(next_page, callback=self.parse)
+ for href in response.css('li.next a::attr(href)'):
+ yield response.follow(href, self.parse)
def parse_author(self, response):
def extract_with_css(query):
@@ -592,6 +616,9 @@ This spider will start from the main page, it will follow all the links to the
authors pages calling the ``parse_author`` callback for each of them, and also
the pagination links with the ``parse`` callback as we saw before.
+Here we're passing callbacks to ``response.follow`` as positional arguments
+to make the code shorter; it also works for ``scrapy.Request``.
+
The ``parse_author`` callback defines a helper function to extract and cleanup the
data from a CSS query and yields the Python dict with the author data.
@@ -652,8 +679,7 @@ with a specific tag, building the URL based on the argument::
next_page = response.css('li.next a::attr(href)').extract_first()
if next_page is not None:
- next_page = response.urljoin(next_page)
- yield scrapy.Request(next_page, self.parse)
+ yield response.follow(next_page, self.parse)
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 1fdd26043..71050fddd 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -683,6 +683,10 @@ TextResponse objects
response.css('p')
+ .. method:: TextResponse.follow(url, ...)
+
+ Return a scrapy.Request instance to follow a link ``url``.
+
.. method:: TextResponse.body_as_unicode()
The same as :attr:`text`, but available as a method. This method is
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 5a6507aa8..1718b1f3b 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst
import six
from six.moves.urllib.parse import urljoin
+import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
html_body_declared_encoding, http_content_type_encoding
+
+from scrapy.link import Link
+from scrapy.http.request import Request
from scrapy.http.response import Response
from scrapy.utils.response import get_base_url
from scrapy.utils.python import memoizemethod_noargs, to_native_str
@@ -116,3 +120,58 @@ class TextResponse(Response):
def css(self, query):
return self.selector.css(query)
+
+ def follow(self, url, callback=None, method='GET', headers=None, body=None,
+ cookies=None, meta=None, encoding=None, priority=0,
+ dont_filter=False, errback=None):
+ # type: (...) -> Request
+ """
+ Return a scrapy.Request instance to follow a link ``url``.
+
+ ``url`` can be:
+
+ * absolute URL;
+ * relative URL;
+ * scrapy.link.Link object (e.g. a link extractor result);
+ * attribute Selector (not SelectorList) - e.g.
+ ``response.css('a::attr(href)')[0]`` or
+ ``response.xpath('//img/@src')[0]``.
+ * a Selector for ``<a>`` element, e.g.
+ ``response.css('a.my_link')[0]``.
+ """
+ if isinstance(url, Link):
+ url = url.url
+ elif isinstance(url, parsel.Selector):
+ url = _url_from_selector(url)
+ elif isinstance(url, parsel.SelectorList):
+ raise ValueError("Please pass either string")
+
+
+ encoding = self.encoding if encoding is None else encoding
+ url = self.urljoin(url)
+ return Request(url, callback,
+ method=method,
+ headers=headers,
+ body=body,
+ cookies=cookies,
+ meta=meta,
+ encoding=encoding,
+ priority=priority,
+ dont_filter=dont_filter,
+ errback=errback)
+
+
+def _url_from_selector(sel):
+ # type: (parsel.Selector) -> str
+ if isinstance(sel.root, six.string_types):
+ # e.g. ::attr(href) result
+ return sel.root
+ if not hasattr(sel.root, 'tag'):
+ raise ValueError("Unsupported selector: %s" % sel)
+ if sel.root.tag != 'a':
+ raise ValueError("Only <a> elements are supported; got <%s>" %
+ sel.root.tag)
+ href = sel.root.get('href')
+ if href is None:
+ raise ValueError("<a> element has no href attribute: %s" % sel)
+ return href