Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

periodic_log: extension updated

Georgiy Zatserklianyi84fb0edd5fc9
diff --git a/scrapy/extensions/logstats_extended.py b/scrapy/extensions/logstats_extended.py
index 7cba0dc2b..63703dace 100644
--- a/scrapy/extensions/logstats_extended.py
+++ b/scrapy/extensions/logstats_extended.py
@@ -14,26 +14,57 @@ class LogStatsExtended:
"""Log basic scraping stats periodically"""
def __init__(
- self, stats, interval=60.0, extended=False, ext_include=None, ext_exclude=None
+ self, stats, interval=60.0,
+ ext_stats_enabled=False,
+ ext_stats_include=None,
+ ext_stats_exclude=None,
+ ext_delta_enabled=False,
+ ext_delta_include=None,
+ ext_delta_exclude=None,
+ ext_timing_enabled=False,
+ ext_timing_include=None,
+ ext_timing_exclude=None,
):
self.stats = stats
self.interval = interval
self.multiplier = 60.0 / self.interval
self.task = None
- self.extended = extended
self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
- self.ext_include = ext_include
- self.ext_exclude = ext_exclude
+ self.ext_stats_enabled = ext_stats_enabled
+ self.ext_stats_include = ext_stats_include
+ self.ext_stats_exclude = ext_stats_exclude
+ self.ext_delta_enabled = ext_delta_enabled
+ self.ext_delta_include = ext_delta_include
+ self.ext_delta_exclude = ext_delta_exclude
+ self.ext_timing_enabled = ext_timing_enabled
+ self.ext_timing_include = ext_timing_include
+ self.ext_timing_exclude = ext_timing_exclude
+
@classmethod
def from_crawler(cls, crawler):
interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
- extended = crawler.settings.getbool("LOGSTATS_EXTENDED_ENABLED")
- ext_include = crawler.settings.getlist("LOGSTATS_EXTENDED_INCLUDE", [])
- ext_exclude = crawler.settings.getlist("LOGSTATS_EXTENDED_EXCLUDE", [])
+ ext_stats_enabled = crawler.settings.getbool("LOGSTATS_EXT_STATS_ENABLED")
+ ext_stats_include = crawler.settings.getlist("LOGSTATS_EXT_STATS_INCLUDE", [])
+ ext_stats_exclude = crawler.settings.getlist("LOGSTATS_EXT_STATS_EXCLUDE", [])
+ ext_delta_enabled = crawler.settings.getbool("LOGSTATS_EXT_DELTA_ENABLED")
+ ext_delta_include = crawler.settings.getlist("LOGSTATS_EXT_DELTA_INCLUDE", [])
+ ext_delta_exclude = crawler.settings.getlist("LOGSTATS_EXT_DELTA_EXCLUDE", [])
+ ext_timing_enabled = crawler.settings.getbool("LOGSTATS_EXT_TIMING_ENABLED")
if not interval:
raise NotConfigured
- o = cls(crawler.stats, interval, extended, ext_include, ext_exclude)
+ if not (ext_stats_enabled or ext_delta_enabled or ext_timing_enabled):
+ raise NotConfigured
+ o = cls(crawler.stats,
+ interval,
+ ext_stats_enabled,
+ ext_stats_include,
+ ext_stats_exclude,
+ ext_delta_enabled,
+ ext_delta_include,
+ ext_delta_exclude,
+ ext_timing_enabled,
+ )
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
@@ -48,17 +79,19 @@ class LogStatsExtended:
def log(self, spider):
data = {}
- data.update(self.log_timing())
- data.update(self.log_delta())
- data.update(self.log_crawler_stats())
+ if self.ext_timing_enabled:
+ data.update(self.log_timing())
+ if self.ext_delta_enabled:
+ data.update(self.log_delta())
+ if self.ext_stats_enabled:
+ data.update(self.log_crawler_stats())
logger.info(self.encoder.encode(data))
-
def log_delta(self):
num_stats = {
k: v
for k, v in self.stats._stats.items()
- if isinstance(v, (int, float)) and self.delta_param_allowed(k)
+ if isinstance(v, (int, float)) and self.param_allowed(k,self.ext_delta_include,self.ext_delta_exclude)
}
delta = {k: v - self.delta_prev.get(k, 0) for k, v in num_stats.items()}
self.delta_prev = num_stats
@@ -76,37 +109,24 @@ class LogStatsExtended:
self.time_prev = now
return {"time": time}
- def log_time(self):
- num_stats = {
+ def log_crawler_stats(self):
+ stats = {
k: v
for k, v in self.stats._stats.items()
- if isinstance(v, (int, float)) and self.delta_param_allowed(k)
- }
- delta = {k: v - self.stats_prev.get(k, 0) for k, v in num_stats.items()}
- self.stats_prev = num_stats
- return {"delta": delta}
+ if self.param_allowed(k,self.ext_stats_include, self.ext_stats_exclude)}
+ return {"stats": stats}
- def log_crawler_stats(self):
- return {"stats": self.stats.get_stats()}
-
- def delta_param_allowed(self, stat_name):
- for p in self.ext_exclude:
+ def param_allowed(self, stat_name, include, exclude):
+ for p in exclude:
if p in stat_name:
return False
- for p in self.ext_include:
+ for p in include:
if p in stat_name:
return True
- if self.ext_include:
- return False
- else:
- return True
+ return False
- def spider_closed(self, spider, reason):
- data = {}
- data.update(self.log_timing())
- data.update(self.log_delta())
- data.update(self.log_crawler_stats())
- logger.info(self.encoder.encode(data))
+ def spider_closed(self, spider, reason):
+ self.log(spider)
if self.task and self.task.running:
- self.task.stop()
\ No newline at end of file
+ self.task.stop()