Scrapy, a fast high-level web crawling & scraping framework for Python.
[MRG+1] Show elapsed time in statscollector (#3638)
Maram Sumanthaa46e1995cd5
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.pyindex 3d9a307b7..8cc5e18ac 100644--- a/scrapy/extensions/corestats.py+++ b/scrapy/extensions/corestats.py@@ -24,7 +24,11 @@ class CoreStats(object):self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)def spider_closed(self, spider, reason):- self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)+ finish_time = datetime.datetime.utcnow()+ elapsed_time = finish_time - self.stats.get_value('start_time')+ elapsed_time_seconds = elapsed_time.total_seconds()+ self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)+ self.stats.set_value('finish_time', finish_time, spider=spider)self.stats.set_value('finish_reason', reason, spider=spider)def item_scraped(self, item, spider):diff --git a/tests/test_closespider.py b/tests/test_closespider.pyindex 0eb1b7944..4a56425b7 100644--- a/tests/test_closespider.py+++ b/tests/test_closespider.py@@ -53,9 +53,5 @@ class TestCloseSpider(TestCase):yield crawler.crawl(total=1000000, mockserver=self.mockserver)reason = crawler.spider.meta['close_reason']self.assertEqual(reason, 'closespider_timeout')- stats = crawler.stats- start = stats.get_value('start_time')- stop = stats.get_value('finish_time')- diff = stop - start- total_seconds = diff.seconds + diff.microseconds+ total_seconds = crawler.stats.get_value('elapsed_time_seconds')self.assertTrue(total_seconds >= close_on)