Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[MRG+1] Show elapsed time in statscollector (#3638)

Maram Sumanthaa46e1995cd5
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 3d9a307b7..8cc5e18ac 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -24,7 +24,11 @@ class CoreStats(object):
self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider)
def spider_closed(self, spider, reason):
- self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider)
+ finish_time = datetime.datetime.utcnow()
+ elapsed_time = finish_time - self.stats.get_value('start_time')
+ elapsed_time_seconds = elapsed_time.total_seconds()
+ self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider)
+ self.stats.set_value('finish_time', finish_time, spider=spider)
self.stats.set_value('finish_reason', reason, spider=spider)
def item_scraped(self, item, spider):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 0eb1b7944..4a56425b7 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -53,9 +53,5 @@ class TestCloseSpider(TestCase):
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
reason = crawler.spider.meta['close_reason']
self.assertEqual(reason, 'closespider_timeout')
- stats = crawler.stats
- start = stats.get_value('start_time')
- stop = stats.get_value('finish_time')
- diff = stop - start
- total_seconds = diff.seconds + diff.microseconds
+ total_seconds = crawler.stats.get_value('elapsed_time_seconds')
self.assertTrue(total_seconds >= close_on)