Scrapy, a fast high-level web crawling & scraping framework for Python.
periodic_log: extension updated
Georgiy Zatserklianyi84fb0edd5fc9
diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.pyindex 7cba0dc2b..63703dace 100644--- a/scrapy/extensions/logstats_extended.py+++ b/scrapy/extensions/logstats_extended.py@@ -14,26 +14,57 @@ class LogStatsExtended:"""Log basic scraping stats periodically"""def __init__(- self, stats, interval=60.0, extended=False, ext_include=None, ext_exclude=None+ self, stats, interval=60.0,+ ext_stats_enabled=False,+ ext_stats_include=None,+ ext_stats_exclude=None,+ ext_delta_enabled=False,+ ext_delta_include=None,+ ext_delta_exclude=None,+ ext_timing_enabled=False,+ ext_timing_include=None,+ ext_timing_exclude=None,):self.stats = statsself.interval = intervalself.multiplier = 60.0 / self.intervalself.task = None- self.extended = extendedself.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)- self.ext_include = ext_include- self.ext_exclude = ext_exclude+ self.ext_stats_enabled = ext_stats_enabled+ self.ext_stats_include = ext_stats_include+ self.ext_stats_exclude = ext_stats_exclude+ self.ext_delta_enabled = ext_delta_enabled+ self.ext_delta_include = ext_delta_include+ self.ext_delta_exclude = ext_delta_exclude+ self.ext_timing_enabled = ext_timing_enabled+ self.ext_timing_include = ext_timing_include+ self.ext_timing_exclude = ext_timing_exclude+@classmethoddef from_crawler(cls, crawler):interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")- extended = crawler.settings.getbool("LOGSTATS_EXTENDED_ENABLED")- ext_include = crawler.settings.getlist("LOGSTATS_EXTENDED_INCLUDE", [])- ext_exclude = crawler.settings.getlist("LOGSTATS_EXTENDED_EXCLUDE", [])+ ext_stats_enabled = crawler.settings.getbool("LOGSTATS_EXT_STATS_ENABLED")+ ext_stats_include = crawler.settings.getlist("LOGSTATS_EXT_STATS_INCLUDE", [])+ ext_stats_exclude = crawler.settings.getlist("LOGSTATS_EXT_STATS_EXCLUDE", [])+ ext_delta_enabled = crawler.settings.getbool("LOGSTATS_EXT_DELTA_ENABLED")+ ext_delta_include = crawler.settings.getlist("LOGSTATS_EXT_DELTA_INCLUDE", [])+ ext_delta_exclude = crawler.settings.getlist("LOGSTATS_EXT_DELTA_EXCLUDE", [])+ ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED")if not interval:raise NotConfigured- o = cls(crawler.stats, interval, extended, ext_include, ext_exclude)+ if not (ext_stats_enabled or ext_delta_enabled or ext_timing_enabled):+ raise NotConfigured+ o = cls(crawler.stats,+ interval,+ ext_stats_enabled,+ ext_stats_include,+ ext_stats_exclude,+ ext_delta_enabled,+ ext_delta_include,+ ext_delta_exclude,+ ext_timing_enabled,+ )crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)return o@@ -48,17 +79,19 @@ class LogStatsExtended:def log(self, spider):data = {}- data.update(self.log_timing())- data.update(self.log_delta())- data.update(self.log_crawler_stats())+ if self.ext_timing_enabled:+ data.update(self.log_timing())+ if self.ext_delta_enabled:+ data.update(self.log_delta())+ if self.ext_stats_enabled:+ data.update(self.log_crawler_stats())logger.info(self.encoder.encode(data))-def log_delta(self):num_stats = {k: vfor k, v in self.stats._stats.items()- if isinstance(v, (int, float)) and self.delta_param_allowed(k)+ if isinstance(v, (int, float)) and self.param_allowed(k,self.ext_delta_include,self.ext_delta_exclude)}delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()}self.delta_prev = num_stats@@ -76,37 +109,24 @@ class LogStatsExtended:self.time_prev = nowreturn {"time": time}- def log_time(self):- num_stats = {+ def log_crawler_stats(self):+ stats = {k: vfor k, v in self.stats._stats.items()- if isinstance(v, (int, float)) and self.delta_param_allowed(k)- }- delta = {k: v - self.stats_prev.get(k, 0) for k, v in num_stats.items()}- self.stats_prev = num_stats- return {"delta": delta}+ if self.param_allowed(k,self.ext_stats_include, self.ext_stats_exclude)}+ return {"stats": stats}- def log_crawler_stats(self):- return {"stats": self.stats.get_stats()}-- def delta_param_allowed(self, stat_name):- for p in self.ext_exclude:+ def param_allowed(self, stat_name, include, exclude):+ for p in exclude:if p in stat_name:return False- for p in self.ext_include:+ for p in include:if p in stat_name:return True- if self.ext_include:- return False- else:- return True+ return False- def spider_closed(self, spider, reason):- data = {}- data.update(self.log_timing())- data.update(self.log_delta())- data.update(self.log_crawler_stats())- logger.info(self.encoder.encode(data))+ def spider_closed(self, spider, reason):+ self.log(spider)if self.task and self.task.running:- self.task.stop()\ No newline at end of file+ self.task.stop()