Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into spider.parse
Eugenio Lacuesta2be2bdd2df4f
diff --cc docs/topics/spiders.rstindex bf410c03f,d4d6e2ea0..e50e4aa0a--- a/docs/topics/spiders.rst+++ b/docs/topics/spiders.rst@@@ -362,14 -360,12 +360,13 @@@ CrawlSpideThis spider also exposes an overrideable method:- .. method:: parse_start_url(response)+ .. method:: parse_start_url(response, **kwargs)- This method is called for the start_urls responses. It allows to parse+ This method is called for each response produced for the URLs in+ the spider's ``start_urls`` attribute. It allows to parse- the initial responses and must return either an item- (:class:`scrapy.item.Item` or :class:`dict`),- a :class:`~scrapy.http.Request`,- or an iterable containing any of them.+ the initial responses and must return either an+ :ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`+ object, or an iterable containing any of them.Crawling rules~~~~~~~~~~~~~~@@@ -550,14 -541,9 +547,14 @@@ XMLFeedSpidespider, and it's intended to perform any last time processing requiredbefore returning the results to the framework core, for example setting theitem IDs. It receives a list of results and the response which originated- those results. It must return a list of results (Items or Requests).+ those results. It must return a list of results (items or requests).+.. warning:: Because of its internal implementation, you must explicitly set+ callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;+ unexpected behaviour can occur otherwise.++XMLFeedSpider example~~~~~~~~~~~~~~~~~~~~~diff --cc tests/spiders.pyindex 8a13c50c7,a360d8206..3eb681819--- a/tests/spiders.py+++ b/tests/spiders.py@@@ -290,5 -275,41 +302,38 @@@ class CrawlSpiderWithErrback(CrawlSpideurl = self.mockserver.url("/alpayload")yield Request(url, method="POST", body=test_body)- def callback(self, response):- self.logger.info('[callback] status %i', response.status)-def errback(self, failure):self.logger.info('[errback] status %i', failure.value.response.status)+++ class BytesReceivedCallbackSpider(MetaSpider):++ full_response_length = 2**18++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ crawler.signals.connect(spider.bytes_received, signals.bytes_received)+ return spider++ def start_requests(self):+ body = b"a" * self.full_response_length+ url = self.mockserver.url("/alpayload")+ yield Request(url, method="POST", body=body, errback=self.errback)++ def parse(self, response):+ self.meta["response"] = response++ def errback(self, failure):+ self.meta["failure"] = failure++ def bytes_received(self, data, request, spider):+ self.meta["bytes_received"] = data+ raise StopDownload(fail=False)+++ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):++ def bytes_received(self, data, request, spider):+ self.meta["bytes_received"] = data+ raise StopDownload(fail=True)diff --cc tests/test_crawl.pyindex 1d5075d22,df920f2a2..642c24651--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@@ -19,8 -26,9 +26,10 @@@ from tests.spiders importAsyncDefAsyncioSpider,AsyncDefSpider,BrokenStartRequestsSpider,+ BytesReceivedCallbackSpider,+ BytesReceivedErrbackSpider,CrawlSpiderWithErrback,+ CrawlSpiderWithParseMethod,DelaySpider,DuplicateStartRequestsSpider,FollowAllSpider,