SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Bug fixes
smicallefab03a299c14e
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.pyindex b167e518..d4abf786 100644--- a/modules/sfp_binaryedge.py+++ b/modules/sfp_binaryedge.py@@ -108,7 +108,7 @@ class sfp_binaryedge(SpiderFootPlugin):headers = {'X-Key': self.opts['binaryedge_api_key']}- url = binaryedgeurl + queryurl.format(qry.encode('utf-8', errors='replace'), page)+ url = binaryedgeurl + queryurl.format(qry, page)res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot", headers=headers)diff --git a/modules/sfp_censys.py b/modules/sfp_censys.pyindex df3f5b39..3ee61a1b 100644--- a/modules/sfp_censys.py+++ b/modules/sfp_censys.py@@ -70,11 +70,12 @@ class sfp_censys(SpiderFootPlugin):secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret']b64_val = base64.b64encode(secret.encode('utf-8'))+headers = {- 'Authorization': 'Basic %s' % b64_val+ 'Authorization': 'Basic %s' % b64_val.decode('utf-8')}- res = self.sf.fetchUrl('https://censys.io/api/v1/view/' + querytype.format(qry.encode('utf-8', errors='replace')),+ res = self.sf.fetchUrl('https://censys.io/api/v1/view/' + querytype.format(qry),timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot",headers=headers)diff --git a/modules/sfp_circllu.py b/modules/sfp_circllu.pyindex 60734821..d07fcfd8 100644--- a/modules/sfp_circllu.py+++ b/modules/sfp_circllu.py@@ -80,7 +80,7 @@ class sfp_circllu(SpiderFootPlugin):secret = self.opts['api_key_login'] + ':' + self.opts['api_key_password']b64_val = base64.b64encode(secret.encode('utf-8'))headers = {- 'Authorization': 'Basic %s' % b64_val+ 'Authorization': 'Basic %s' % b64_val.decode('utf-8')}# Be more forgiving with the timeout as some queries for subnets can be slowdiff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex ba5b7113..30487160 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -62,7 +62,7 @@ class sfp_email(SpiderFootPlugin):email = email.lower()# Get the domain and strip potential ending .- mailDom = email.lower().split('@')[1].strip('.')+ mailDom = email.split('@')[1].strip('.')if not self.sf.validHost(mailDom, self.opts['_internettlds']):self.sf.debug("Skipping " + email + " as not a valid e-mail.")return Nonediff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.pyindex e03195a5..c433a822 100644--- a/modules/sfp_fringeproject.py+++ b/modules/sfp_fringeproject.py@@ -137,7 +137,12 @@ class sfp_fringeproject(SpiderFootPlugin):continuefor tag in tags:- port = re.findall(r'^port:([0-9]+)', tag)+ try:+ port = re.findall(r'^port:([0-9]+)', tag)+ except BaseException as e:+ self.sf.debug("Didn't get sane data from FringeProject.")+ continue+if len(port) > 0:evt = SpiderFootEvent('TCP_PORT_OPEN', value + ':' + str(port[0]), self.__name__, event)self.notifyListeners(evt)diff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.pyindex b4e998b1..eabb81bc 100644--- a/modules/sfp_haveibeenpwned.py+++ b/modules/sfp_haveibeenpwned.py@@ -119,10 +119,15 @@ class sfp_haveibeenpwned(SpiderFootPlugin):return Nonefor n in data:- if not self.opts['api_key']:- site = n["Title"]- else:- site = n["Name"]+ try:+ if not self.opts['api_key']:+ site = n["Title"]+ else:+ site = n["Name"]+ except BaseException as e:+ self.sf.debug("Unable to parse result from HaveIBeenPwned?")+ continue+evt = eventName + "_COMPROMISED"# Notify other modules of what you've founde = SpiderFootEvent(evt, eventData + " [" + site + "]",diff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.pyindex f67e36bd..8a736f8b 100644--- a/modules/sfp_psbdmp.py+++ b/modules/sfp_psbdmp.py@@ -49,7 +49,7 @@ class sfp_psbdmp(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- ret = ["LEAKSITE_URL"]+ ret = ["LEAKSITE_URL", "LEAKSITE_CONTENT"]return retdiff --git a/modules/sfp_threatminer.py b/modules/sfp_threatminer.pyindex 227443b2..c60464e7 100644--- a/modules/sfp_threatminer.py+++ b/modules/sfp_threatminer.py@@ -83,7 +83,7 @@ class sfp_threatminer(SpiderFootPlugin):queryurl = "/v2/" + tgttype + ".php?q={0}&rt=2"threatminerurl = "https://api.threatminer.org"- url = threatminerurl + queryurl.format(qry.encode('utf-8', errors='replace'))+ url = threatminerurl + queryurl.format(qry)res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")if res['content'] is None:diff --git a/sflib.py b/sflib.pyindex c1ed068e..924d8993 100644--- a/sflib.py+++ b/sflib.py@@ -128,9 +128,9 @@ class SpiderFoot:session = self.getSession()res = session.get(val)if splitLines:- return res.content.splitlines()+ return res.content.decode('utf-8').splitlines()else:- return res.content+ return res.content.decode('utf-8')except BaseException as e:if fatal:self.error("Unable to open option URL, " + val + ": " + str(e))