Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into spider.parse

Eugenio Lacuesta2be2bdd2df4f
diff --cc docs/topics/spiders.rst
index bf410c03f,d4d6e2ea0..e50e4aa0a
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@@ -362,14 -360,12 +360,13 @@@ CrawlSpide
This spider also exposes an overrideable method:
- .. method:: parse_start_url(response)
+ .. method:: parse_start_url(response, **kwargs)
- This method is called for the start_urls responses. It allows to parse
+ This method is called for each response produced for the URLs in
+ the spider's ``start_urls`` attribute. It allows to parse
- the initial responses and must return either an item
- (:class:`scrapy.item.Item` or :class:`dict`),
- a :class:`~scrapy.http.Request`,
- or an iterable containing any of them.
+ the initial responses and must return either an
+ :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
+ object, or an iterable containing any of them.
Crawling rules
~~~~~~~~~~~~~~
@@@ -550,14 -541,9 +547,14 @@@ XMLFeedSpide
spider, and it's intended to perform any last time processing required
before returning the results to the framework core, for example setting the
item IDs. It receives a list of results and the response which originated
- those results. It must return a list of results (Items or Requests).
+ those results. It must return a list of results (items or requests).
+.. warning:: Because of its internal implementation, you must explicitly set
+ callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
+ unexpected behaviour can occur otherwise.
+
+
XMLFeedSpider example
~~~~~~~~~~~~~~~~~~~~~
diff --cc tests/spiders.py
index 8a13c50c7,a360d8206..3eb681819
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@@ -290,5 -275,41 +302,38 @@@ class CrawlSpiderWithErrback(CrawlSpide
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
- def callback(self, response):
- self.logger.info('[callback] status %i', response.status)
-
def errback(self, failure):
self.logger.info('[errback] status %i', failure.value.response.status)
+
+
+ class BytesReceivedCallbackSpider(MetaSpider):
+
+ full_response_length = 2**18
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ crawler.signals.connect(spider.bytes_received, signals.bytes_received)
+ return spider
+
+ def start_requests(self):
+ body = b"a" * self.full_response_length
+ url = self.mockserver.url("/alpayload")
+ yield Request(url, method="POST", body=body, errback=self.errback)
+
+ def parse(self, response):
+ self.meta["response"] = response
+
+ def errback(self, failure):
+ self.meta["failure"] = failure
+
+ def bytes_received(self, data, request, spider):
+ self.meta["bytes_received"] = data
+ raise StopDownload(fail=False)
+
+
+ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
+
+ def bytes_received(self, data, request, spider):
+ self.meta["bytes_received"] = data
+ raise StopDownload(fail=True)
diff --cc tests/test_crawl.py
index 1d5075d22,df920f2a2..642c24651
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@@ -19,8 -26,9 +26,10 @@@ from tests.spiders import
AsyncDefAsyncioSpider,
AsyncDefSpider,
BrokenStartRequestsSpider,
+ BytesReceivedCallbackSpider,
+ BytesReceivedErrbackSpider,
CrawlSpiderWithErrback,
+ CrawlSpiderWithParseMethod,
DelaySpider,
DuplicateStartRequestsSpider,
FollowAllSpider,