Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve test coverage of Scraper.

Andrey Rakhmatullin23c206af35a8
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 2942dfa58..6f69d668e 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -374,32 +374,24 @@ class Scraper:
errors that got propagated thru here).
spider_failure: the value passed into the errback of self.call_spider()
+ (likely raised in the request errback)
+
download_failure: the value passed into _scrape2() from
ExecutionEngine._handle_downloader_output() as "result"
+ (likely raised in the download handler or a downloader middleware)
"""
if not download_failure.check(IgnoreRequest):
assert self.crawler.spider
- if download_failure.frames:
- logkws = self.logformatter.download_error(
- download_failure, request, self.crawler.spider
- )
- logger.log(
- *logformatter_adapter(logkws),
- extra={"spider": self.crawler.spider},
- exc_info=failure_to_exc_info(download_failure),
- )
- else:
- errmsg = download_failure.getErrorMessage()
- if errmsg:
- logkws = self.logformatter.download_error(
- download_failure, request, self.crawler.spider, errmsg
- )
- logger.log(
- *logformatter_adapter(logkws),
- extra={"spider": self.crawler.spider},
- )
-
+ logkws = self.logformatter.download_error(
+ download_failure, request, self.crawler.spider
+ )
+ logger.log(
+ *logformatter_adapter(logkws),
+ extra={"spider": self.crawler.spider},
+ exc_info=failure_to_exc_info(download_failure),
+ )
if spider_failure is not download_failure:
+ # a request errback raised a different exception, it needs to be handled later
return spider_failure
return None
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index f49deac1f..a8174d537 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
import json
import logging
import unittest
@@ -723,3 +725,78 @@ class TestCrawlSpider(TestCase):
assert crawler.spider.meta[
"failure"
].value.response.headers == crawler.spider.meta.get("headers_received")
+
+ @defer.inlineCallbacks
+ def test_spider_errback(self):
+ failures = []
+
+ def eb(failure: Failure) -> Failure:
+ failures.append(failure)
+ return failure
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+ assert len(failures) == 1
+ assert "HTTP status code is not handled or not allowed" in str(log)
+ assert "Spider error processing" not in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_silence(self):
+ failures = []
+
+ def eb(failure: Failure) -> None:
+ failures.append(failure)
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+ assert len(failures) == 1
+ assert "HTTP status code is not handled or not allowed" not in str(log)
+ assert "Spider error processing" not in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_exception(self):
+ def eb(failure: Failure) -> None:
+ raise ValueError("foo")
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/status?n=400"), errback_func=eb
+ )
+ assert "Spider error processing" in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_downloader_error(self):
+ failures = []
+
+ def eb(failure: Failure) -> Failure:
+ failures.append(failure)
+ return failure
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+ )
+ assert len(failures) == 1
+ assert "Error downloading" in str(log)
+ assert "Spider error processing" not in str(log)
+
+ @defer.inlineCallbacks
+ def test_spider_errback_exception_downloader_error(self):
+ def eb(failure: Failure) -> None:
+ raise ValueError("foo")
+
+ crawler = get_crawler(SingleRequestSpider)
+ with LogCapture() as log:
+ yield crawler.crawl(
+ seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+ )
+ assert "Error downloading" in str(log)
+ assert "Spider error processing" in str(log)