Scrapy, a fast high-level web crawling & scraping framework for Python.
StopDownload: check partial response contents
Eugenio Lacuestafbe915d9cbd8
diff --git a/tests/spiders.py b/tests/spiders.pyindex ebe81a1a9..05078cc04 100644--- a/tests/spiders.py+++ b/tests/spiders.py@@ -293,10 +293,12 @@ class BytesReceivedCallbackSpider(MetaSpider):self.meta["failure"] = failuredef bytes_received(self, data, request, spider):+ self.meta["bytes_received"] = dataraise StopDownload(fail=False)class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):def bytes_received(self, data, request, spider):+ self.meta["bytes_received"] = dataraise StopDownload(fail=True)diff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex 5e70064c2..e2426e0da 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -478,7 +478,8 @@ with multiples linesyield crawler.crawl(mockserver=self.mockserver)self.assertIsNone(crawler.spider.meta.get("failure"))self.assertIsInstance(crawler.spider.meta["response"], Response)- self.assertLess(len(crawler.spider.meta["response"].text), crawler.spider.full_response_length)+ self.assertEqual(crawler.spider.meta["response"].body, crawler.spider.meta.get("bytes_received"))+ self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)@defer.inlineCallbacksdef test_stop_download_errback(self):@@ -488,4 +489,5 @@ with multiples linesself.assertIsInstance(crawler.spider.meta["failure"], Failure)self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)self.assertIsInstance(crawler.spider.meta["failure"].response, Response)- self.assertLess(len(crawler.spider.meta["failure"].response.text), crawler.spider.full_response_length)+ self.assertEqual(crawler.spider.meta["failure"].response.body, crawler.spider.meta.get("bytes_received"))+ self.assertLess(len(crawler.spider.meta["failure"].response.body), crawler.spider.full_response_length)