Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

feat: Add count to spider_exceptions stats (#6740)

Laerte Pereira2ee01efe496d
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index b664b61f6..496adb500 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -246,6 +246,7 @@ class Scraper:
spider=spider,
)
assert self.crawler.stats
+ self.crawler.stats.inc_value("spider_exceptions/count", spider=spider)
self.crawler.stats.inc_value(
f"spider_exceptions/{_failure.value.__class__.__name__}", spider=spider
)
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 476662789..4a17b254b 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -88,6 +88,7 @@ class TestCloseSpider(TestCase):
assert reason == "closespider_errorcount"
key = f"spider_exceptions/{crawler.spider.exception_cls.__name__}"
errorcount = crawler.stats.get_value(key)
+ assert crawler.stats.get_value("spider_exceptions/count") >= close_on
assert errorcount >= close_on
@defer.inlineCallbacks