Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

modules: Log record date/time for skipped records which are too old (#1459)

bcoles35a1696aabe8
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py
index a950c222..7083e894 100644
--- a/modules/sfp_binaryedge.py
+++ b/modules/sfp_binaryedge.py
@@ -343,7 +343,7 @@ class sfp_binaryedge(SpiderFootPlugin):
age_limit_ts = int(time.time()) - (86400 * self.opts['torrent_age_limit_days'])
if self.opts['torrent_age_limit_days'] > 0 and created_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({created_ts}), skipping.")
continue
dat = "Torrent: " + rec.get("torrent", "???").get("name") + " @ " + rec.get('torrent').get("source", "???")
@@ -380,7 +380,7 @@ class sfp_binaryedge(SpiderFootPlugin):
age_limit_ts = int(time.time()) - (86400 * self.opts['cve_age_limit_days'])
if self.opts['cve_age_limit_days'] > 0 and created_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({created_ts}), skipping.")
continue
cves = rec.get('cves')
@@ -419,7 +419,7 @@ class sfp_binaryedge(SpiderFootPlugin):
age_limit_ts = int(time.time()) - (86400 * self.opts['port_age_limit_days'])
if self.opts['port_age_limit_days'] > 0 and created_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({created_ts}), skipping.")
continue
port = str(prec['target']['port'])
diff --git a/modules/sfp_circllu.py b/modules/sfp_circllu.py
index aa7aeef1..d7cef243 100644
--- a/modules/sfp_circllu.py
+++ b/modules/sfp_circllu.py
@@ -209,7 +209,7 @@ class sfp_circllu(SpiderFootPlugin):
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts['age_limit_days'] > 0 and rec['time_last'] < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({rec['time_last']}), skipping.")
continue
cohosts = list()
diff --git a/modules/sfp_dnsdb.py b/modules/sfp_dnsdb.py
index 7fa09e18..086608db 100644
--- a/modules/sfp_dnsdb.py
+++ b/modules/sfp_dnsdb.py
@@ -141,7 +141,7 @@ class sfp_dnsdb(SpiderFootPlugin):
def isTooOld(self, lastSeen):
ageLimitTs = int(time.time()) - (86400 * self.opts["age_limit_days"])
if self.opts["age_limit_days"] > 0 and lastSeen < ageLimitTs:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({lastSeen}), skipping.")
return True
return False
diff --git a/modules/sfp_fraudguard.py b/modules/sfp_fraudguard.py
index 7471865a..712ca864 100644
--- a/modules/sfp_fraudguard.py
+++ b/modules/sfp_fraudguard.py
@@ -180,7 +180,7 @@ class sfp_fraudguard(SpiderFootPlugin):
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({created_dt}), skipping.")
continue
# For netblocks, we need to create the IP address event so that
diff --git a/modules/sfp_greynoise.py b/modules/sfp_greynoise.py
index b3b48e34..a756514d 100644
--- a/modules/sfp_greynoise.py
+++ b/modules/sfp_greynoise.py
@@ -183,7 +183,7 @@ class sfp_greynoise(SpiderFootPlugin):
lastseen_ts = int(time.mktime(lastseen_dt.timetuple()))
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts['age_limit_days'] > 0 and lastseen_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({lastseen_dt}), skipping.")
return
# Only report meta data about the target, not affiliates
diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.py
index c862b43e..ef890984 100644
--- a/modules/sfp_intelx.py
+++ b/modules/sfp_intelx.py
@@ -225,7 +225,7 @@ class sfp_intelx(SpiderFootPlugin):
try:
last_seen = int(datetime.datetime.strptime(rec['added'].split(".")[0], '%Y-%m-%dT%H:%M:%S').strftime('%s')) * 1000
if last_seen < agelimit:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({last_seen}), skipping.")
continue
val = None
diff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.py
index 68117d80..df9678c0 100644
--- a/modules/sfp_onyphe.py
+++ b/modules/sfp_onyphe.py
@@ -235,7 +235,7 @@ class sfp_onyphe(SpiderFootPlugin):
age_limit_ts = int(time.time()) - (86400 * limit)
if last_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({last_dt}), skipping.")
return False
return True
diff --git a/modules/sfp_threatminer.py b/modules/sfp_threatminer.py
index 23abd254..9ebf5ba3 100644
--- a/modules/sfp_threatminer.py
+++ b/modules/sfp_threatminer.py
@@ -189,7 +189,7 @@ class sfp_threatminer(SpiderFootPlugin):
last_ts = int(time.mktime(last_seen.timetuple()))
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts['age_limit_days'] > 0 and last_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({last_seen}), skipping.")
continue
host = rec['domain']
diff --git a/modules/sfp_xforce.py b/modules/sfp_xforce.py
index f9b937a0..3c6bba84 100644
--- a/modules/sfp_xforce.py
+++ b/modules/sfp_xforce.py
@@ -258,7 +258,7 @@ class sfp_xforce(SpiderFootPlugin):
host = rec['value']
if self.opts['age_limit_days'] > 0 and last_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({last_dt}), skipping.")
continue
if not self.opts["cohostsamedomain"]:
@@ -298,7 +298,7 @@ class sfp_xforce(SpiderFootPlugin):
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({created_dt}), skipping.")
continue
reason = result.get("reason", "")
score = result.get("score", 0)
@@ -350,7 +350,7 @@ class sfp_xforce(SpiderFootPlugin):
host = rec['value']
if self.opts['age_limit_days'] > 0 and last_ts < age_limit_ts:
- self.sf.debug("Record found but too old, skipping.")
+ self.sf.debug(f"Record found but too old ({last_ts}), skipping.")
continue
e = SpiderFootEvent(evtType, entry, self.__name__, event)