Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve test coverage of Scraper.
Andrey Rakhmatullin23c206af35a8
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 2942dfa58..6f69d668e 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -374,32 +374,24 @@ class Scraper:errors that got propagated thru here).spider_failure: the value passed into the errback of self.call_spider()+ (likely raised in the request errback)+download_failure: the value passed into _scrape2() fromExecutionEngine._handle_downloader_output() as "result"+ (likely raised in the download handler or a downloader middleware)"""if not download_failure.check(IgnoreRequest):assert self.crawler.spider- if download_failure.frames:- logkws = self.logformatter.download_error(- download_failure, request, self.crawler.spider- )- logger.log(- *logformatter_adapter(logkws),- extra={"spider": self.crawler.spider},- exc_info=failure_to_exc_info(download_failure),- )- else:- errmsg = download_failure.getErrorMessage()- if errmsg:- logkws = self.logformatter.download_error(- download_failure, request, self.crawler.spider, errmsg- )- logger.log(- *logformatter_adapter(logkws),- extra={"spider": self.crawler.spider},- )-+ logkws = self.logformatter.download_error(+ download_failure, request, self.crawler.spider+ )+ logger.log(+ *logformatter_adapter(logkws),+ extra={"spider": self.crawler.spider},+ exc_info=failure_to_exc_info(download_failure),+ )if spider_failure is not download_failure:+ # a request errback raised a different exception, it needs to be handled laterreturn spider_failurereturn Nonediff --git a/tests/test_crawl.py b/tests/test_crawl.pyindex f49deac1f..a8174d537 100644--- a/tests/test_crawl.py+++ b/tests/test_crawl.py@@ -1,3 +1,5 @@+from __future__ import annotations+import jsonimport loggingimport unittest@@ -723,3 +725,78 @@ class TestCrawlSpider(TestCase):assert crawler.spider.meta["failure"].value.response.headers == crawler.spider.meta.get("headers_received")++ @defer.inlineCallbacks+ def test_spider_errback(self):+ failures = []++ def eb(failure: Failure) -> Failure:+ failures.append(failure)+ return failure++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert len(failures) == 1+ assert "HTTP status code is not handled or not allowed" in str(log)+ assert "Spider error processing" not in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_silence(self):+ failures = []++ def eb(failure: Failure) -> None:+ failures.append(failure)++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert len(failures) == 1+ assert "HTTP status code is not handled or not allowed" not in str(log)+ assert "Spider error processing" not in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_exception(self):+ def eb(failure: Failure) -> None:+ raise ValueError("foo")++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/status?n=400"), errback_func=eb+ )+ assert "Spider error processing" in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_downloader_error(self):+ failures = []++ def eb(failure: Failure) -> Failure:+ failures.append(failure)+ return failure++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert len(failures) == 1+ assert "Error downloading" in str(log)+ assert "Spider error processing" not in str(log)++ @defer.inlineCallbacks+ def test_spider_errback_exception_downloader_error(self):+ def eb(failure: Failure) -> None:+ raise ValueError("foo")++ crawler = get_crawler(SingleRequestSpider)+ with LogCapture() as log:+ yield crawler.crawl(+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb+ )+ assert "Error downloading" in str(log)+ assert "Spider error processing" in str(log)