Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix TypeError when using DummyStatsCollector
Eugenio Lacuestae0fabab5cc74
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex 8cc5e18ac..20adfbe4b 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -1,14 +1,16 @@"""Extension for collecting core stats like items scraped and start/finish times"""-import datetime+from datetime import datetimefrom scrapy import signals+class CoreStats(object):def __init__(self, stats):self.stats = stats+ self.start_time = None@classmethoddef from_crawler(cls, crawler):@@ -21,11 +23,12 @@ class CoreStats(object):return odef spider_opened(self, spider):- self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)+ self.start_time = datetime.utcnow()+ self.stats.set_value('start_time', self.start_time, spider=spider)def spider_closed(self, spider, reason):- finish_time = datetime.datetime.utcnow()- elapsed_time = finish_time - self.stats.get_value('start_time')+ finish_time = datetime.utcnow()+ elapsed_time = finish_time - self.start_timeelapsed_time_seconds = elapsed_time.total_seconds()self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)self.stats.set_value('finish_time', finish_time, spider=spider)diff --git a/tests/test_stats.py b/tests/test_stats.pyindex 9f950ebc9..2033dbe07 100644--- a/tests/test_stats.py+++ b/tests/test_stats.py@@ -1,10 +1,59 @@+from datetime import datetimeimport unittest+try:+ from unittest import mock+except ImportError:+ import mock++from scrapy.extensions.corestats import CoreStatsfrom scrapy.spiders import Spiderfrom scrapy.statscollectors import StatsCollector, DummyStatsCollectorfrom scrapy.utils.test import get_crawler+class CoreStatsExtensionTest(unittest.TestCase):++ def setUp(self):+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider('foo')++ @mock.patch('scrapy.extensions.corestats.datetime')+ def test_core_stats_default_stats_collector(self, mock_datetime):+ fixed_datetime = datetime(2019, 12, 1, 11, 38)+ mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)+ self.crawler.stats = StatsCollector(self.crawler)+ ext = CoreStats.from_crawler(self.crawler)+ ext.spider_opened(self.spider)+ ext.item_scraped({}, self.spider)+ ext.response_received(self.spider)+ ext.item_dropped({}, self.spider, ZeroDivisionError())+ ext.spider_closed(self.spider, 'finished')+ self.assertEqual(+ ext.stats._stats,+ {+ 'start_time': fixed_datetime,+ 'finish_time': fixed_datetime,+ 'item_scraped_count': 1,+ 'response_received_count': 1,+ 'item_dropped_count': 1,+ 'item_dropped_reasons_count/ZeroDivisionError': 1,+ 'finish_reason': 'finished',+ 'elapsed_time_seconds': 0.0,+ }+ )++ def test_core_stats_dummy_stats_collector(self):+ self.crawler.stats = DummyStatsCollector(self.crawler)+ ext = CoreStats.from_crawler(self.crawler)+ ext.spider_opened(self.spider)+ ext.item_scraped({}, self.spider)+ ext.response_received(self.spider)+ ext.item_dropped({}, self.spider, ZeroDivisionError())+ ext.spider_closed(self.spider, 'finished')+ self.assertEqual(ext.stats._stats, {})++class StatsCollectorTest(unittest.TestCase):def setUp(self):