SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Handle parsing errors in files downloaded.
Steve Micallefa8277a9f7e2a
diff --git a/modules/sfp_bambenek.py b/modules/sfp_bambenek.pyindex 19536f26..a24d0641 100644--- a/modules/sfp_bambenek.py+++ b/modules/sfp_bambenek.py@@ -188,14 +188,19 @@ class sfp_bambenek(SpiderFootPlugin):self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")return urlelse:- # Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))- rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url+ try:+ # Check for the domain and the hostname+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))+ for line in data['content'].split('\n'):+ if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \+ re.match(rxTgt, line, re.IGNORECASE):+ self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")+ return url+ except BaseException as e:+ self.sf.debug("Error encountered parsing 2: " + str(e))+ continue+return Nonedef lookupItem(self, resourceId, itemType, target):diff --git a/modules/sfp_cybercrimetracker.py b/modules/sfp_cybercrimetracker.pyindex 2c16ea26..4ae97755 100644--- a/modules/sfp_cybercrimetracker.py+++ b/modules/sfp_cybercrimetracker.py@@ -47,11 +47,11 @@ class sfp_cybercrimetracker(SpiderFootPlugin):# Be sure to completely clear any class variables in setup()# or you run the risk of data persisting between scan runs.- results = list()+ results = dict()def setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = dict()# Clear / reset any other class member variables here# or you risk them persisting between threads.@@ -176,13 +176,17 @@ class sfp_cybercrimetracker(SpiderFootPlugin):return urlelse:# Check for the domain and the hostname- rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))- rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url+ try:+ rxDom = unicode(malchecks[check]['regex']).format(targetDom.decode('string_escape'))+ rxTgt = unicode(malchecks[check]['regex']).format(target.decode('string_escape'))+ for line in data['content'].split('\n'):+ if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \+ re.match(rxTgt, line, re.IGNORECASE):+ self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")+ return url+ except BaseException as e:+ self.sf.debug("Error encountered parsing 2: " + str(e))+ continuereturn Nonedef lookupItem(self, resourceId, itemType, target):@@ -210,7 +214,7 @@ class sfp_cybercrimetracker(SpiderFootPlugin):self.sf.debug("Skipping " + eventData + ", already checked.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Trueif eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):return None