Scrapy, a fast high-level web crawling & scraping framework for Python.
renamed variables to clarify that we are using peak memory and not current memory utilization.
Christopher Gambrellfc8968672a5c
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.pyindex bf2ee4e6d..7bc6564e7 100644--- a/scrapy/extensions/memusage.py+++ b/scrapy/extensions/memusage.py@@ -75,8 +75,8 @@ class MemoryUsage:self.crawler.stats.max_value('memusage/max', self.get_virtual_size())def _check_limit(self):- current_mem_usage = self.get_virtual_size()- if current_mem_usage > self.limit:+ peak_mem_usage = self.get_virtual_size()+ if peak_mem_usage > self.limit:self.crawler.stats.set_value('memusage/limit_reached', 1)mem = self.limit / 1024 / 1024logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",@@ -94,7 +94,7 @@ class MemoryUsage:else:self.crawler.stop()else:- logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024})+ logger.info("Peak memory usage is %(virtualsize)dM", {'virtualsize': peak_mem_usage / 1024 / 1024})def _check_warning(self):if self.warned: # warn only once