Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix the calculate_final_stats method (#6599)

Arthurb10d46d280fb
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index e829d8b92..f2e1f57b8 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -83,12 +83,15 @@ class LogStats:
self, spider: Spider
) -> tuple[None, None] | tuple[float, float]:
start_time = self.stats.get_value("start_time")
- finished_time = self.stats.get_value("finished_time")
+ finish_time = self.stats.get_value("finish_time")
- if not start_time or not finished_time:
+ if not start_time or not finish_time:
return None, None
- mins_elapsed = (finished_time - start_time).seconds / 60
+ mins_elapsed = (finish_time - start_time).seconds / 60
+
+ if mins_elapsed == 0:
+ return None, None
items = self.stats.get_value("item_scraped_count", 0)
pages = self.stats.get_value("response_received_count", 0)
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index d87285df7..a4b002e34 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -47,7 +47,7 @@ class TestLogStats(unittest.TestCase):
# Simulate when spider closes after running for 30 mins
self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
- self.stats.set_value("finished_time", datetime.fromtimestamp(1655101972))
+ self.stats.set_value("finish_time", datetime.fromtimestamp(1655101972))
logstats.spider_closed(self.spider, "test reason")
self.assertEqual(self.stats.get_value("responses_per_minute"), 172.9)
self.assertEqual(self.stats.get_value("items_per_minute"), 116.4)
@@ -60,3 +60,12 @@ class TestLogStats(unittest.TestCase):
logstats.spider_closed(self.spider, "test reason")
self.assertIsNone(self.stats.get_value("responses_per_minute"))
self.assertIsNone(self.stats.get_value("items_per_minute"))
+
+ def test_stats_calculation_no_elapsed_time(self):
+ """The stat values should be None since the elapsed time is 0."""
+ logstats = LogStats.from_crawler(self.crawler)
+ self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
+ self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
+ logstats.spider_closed(self.spider, "test reason")
+ self.assertIsNone(self.stats.get_value("responses_per_minute"))
+ self.assertIsNone(self.stats.get_value("items_per_minute"))