Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Handle parsing errors in files downloaded.

Steve Micallefa8277a9f7e2a
diff --git a/modules/sfp_bambenek.py b/modules/sfp_bambenek.py
index 19536f26..a24d0641 100644
--- a/modules/sfp_bambenek.py
+++ b/modules/sfp_bambenek.py
@@ -188,14 +188,19 @@ class sfp_bambenek(SpiderFootPlugin):
self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
return url
else:
- # Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
- rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
- for line in data['content'].split('\n'):
- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
- re.match(rxTgt, line, re.IGNORECASE):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
+ try:
+ # Check for the domain and the hostname
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
+ for line in data['content'].split('\n'):
+ if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
+ re.match(rxTgt, line, re.IGNORECASE):
+ self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
+ return url
+ except BaseException as e:
+ self.sf.debug("Error encountered parsing 2: " + str(e))
+ continue
+
return None
def lookupItem(self, resourceId, itemType, target):
diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.py
index 2c16ea26..4ae97755 100644
--- a/modules/sfp_cybercrimetracker.py
+++ b/modules/sfp_cybercrimetracker.py
@@ -47,11 +47,11 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
# Be sure to completely clear any class variables in setup()
# or you run the risk of data persisting between scan runs.
- results = list()
+ results = dict()
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = dict()
# Clear / reset any other class member variables here
# or you risk them persisting between threads.
@@ -176,13 +176,17 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
return url
else:
# Check for the domain and the hostname
- rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
- rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
- for line in data['content'].split('\n'):
- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
- re.match(rxTgt, line, re.IGNORECASE):
- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
- return url
+ try:
+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))
+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))
+ for line in data['content'].split('\n'):
+ if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \
+ re.match(rxTgt, line, re.IGNORECASE):
+ self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")
+ return url
+ except BaseException as e:
+ self.sf.debug("Error encountered parsing 2: " + str(e))
+ continue
return None
def lookupItem(self, resourceId, itemType, target):
@@ -210,7 +214,7 @@ class sfp_cybercrimetracker(SpiderFootPlugin):
self.sf.debug("Skipping " + eventData + ", already checked.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
return None