Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Bug fixes

smicallefab03a299c14e
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py
index b167e518..d4abf786 100644
--- a/modules/sfp_binaryedge.py
+++ b/modules/sfp_binaryedge.py
@@ -108,7 +108,7 @@ class sfp_binaryedge(SpiderFootPlugin):
headers = {
'X-Key': self.opts['binaryedge_api_key']
}
- url = binaryedgeurl + queryurl.format(qry.encode('utf-8', errors='replace'), page)
+ url = binaryedgeurl + queryurl.format(qry, page)
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot", headers=headers)
diff --git a/modules/sfp_censys.py b/modules/sfp_censys.py
index df3f5b39..3ee61a1b 100644
--- a/modules/sfp_censys.py
+++ b/modules/sfp_censys.py
@@ -70,11 +70,12 @@ class sfp_censys(SpiderFootPlugin):
secret = self.opts['censys_api_key_uid'] + ':' + self.opts['censys_api_key_secret']
b64_val = base64.b64encode(secret.encode('utf-8'))
+
headers = {
- 'Authorization': 'Basic %s' % b64_val
+ 'Authorization': 'Basic %s' % b64_val.decode('utf-8')
}
- res = self.sf.fetchUrl('https://censys.io/api/v1/view/' + querytype.format(qry.encode('utf-8', errors='replace')),
+ res = self.sf.fetchUrl('https://censys.io/api/v1/view/' + querytype.format(qry),
timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot",
headers=headers)
diff --git a/modules/sfp_circllu.py b/modules/sfp_circllu.py
index 60734821..d07fcfd8 100644
--- a/modules/sfp_circllu.py
+++ b/modules/sfp_circllu.py
@@ -80,7 +80,7 @@ class sfp_circllu(SpiderFootPlugin):
secret = self.opts['api_key_login'] + ':' + self.opts['api_key_password']
b64_val = base64.b64encode(secret.encode('utf-8'))
headers = {
- 'Authorization': 'Basic %s' % b64_val
+ 'Authorization': 'Basic %s' % b64_val.decode('utf-8')
}
# Be more forgiving with the timeout as some queries for subnets can be slow
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index ba5b7113..30487160 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -62,7 +62,7 @@ class sfp_email(SpiderFootPlugin):
email = email.lower()
# Get the domain and strip potential ending .
- mailDom = email.lower().split('@')[1].strip('.')
+ mailDom = email.split('@')[1].strip('.')
if not self.sf.validHost(mailDom, self.opts['_internettlds']):
self.sf.debug("Skipping " + email + " as not a valid e-mail.")
return None
diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.py
index e03195a5..c433a822 100644
--- a/modules/sfp_fringeproject.py
+++ b/modules/sfp_fringeproject.py
@@ -137,7 +137,12 @@ class sfp_fringeproject(SpiderFootPlugin):
continue
for tag in tags:
- port = re.findall(r'^port:([0-9]+)', tag)
+ try:
+ port = re.findall(r'^port:([0-9]+)', tag)
+ except BaseException as e:
+ self.sf.debug("Didn't get sane data from FringeProject.")
+ continue
+
if len(port) > 0:
evt = SpiderFootEvent('TCP_PORT_OPEN', value + ':' + str(port[0]), self.__name__, event)
self.notifyListeners(evt)
diff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.py
index b4e998b1..eabb81bc 100644
--- a/modules/sfp_haveibeenpwned.py
+++ b/modules/sfp_haveibeenpwned.py
@@ -119,10 +119,15 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
return None
for n in data:
- if not self.opts['api_key']:
- site = n["Title"]
- else:
- site = n["Name"]
+ try:
+ if not self.opts['api_key']:
+ site = n["Title"]
+ else:
+ site = n["Name"]
+ except BaseException as e:
+ self.sf.debug("Unable to parse result from HaveIBeenPwned?")
+ continue
+
evt = eventName + "_COMPROMISED"
# Notify other modules of what you've found
e = SpiderFootEvent(evt, eventData + " [" + site + "]",
diff --git a/modules/sfp_psbdmp.py b/modules/sfp_psbdmp.py
index f67e36bd..8a736f8b 100644
--- a/modules/sfp_psbdmp.py
+++ b/modules/sfp_psbdmp.py
@@ -49,7 +49,7 @@ class sfp_psbdmp(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- ret = ["LEAKSITE_URL"]
+ ret = ["LEAKSITE_URL", "LEAKSITE_CONTENT"]
return ret
diff --git a/modules/sfp_threatminer.py b/modules/sfp_threatminer.py
index 227443b2..c60464e7 100644
--- a/modules/sfp_threatminer.py
+++ b/modules/sfp_threatminer.py
@@ -83,7 +83,7 @@ class sfp_threatminer(SpiderFootPlugin):
queryurl = "/v2/" + tgttype + ".php?q={0}&rt=2"
threatminerurl = "https://api.threatminer.org"
- url = threatminerurl + queryurl.format(qry.encode('utf-8', errors='replace'))
+ url = threatminerurl + queryurl.format(qry)
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")
if res['content'] is None:
diff --git a/sflib.py b/sflib.py
index c1ed068e..924d8993 100644
--- a/sflib.py
+++ b/sflib.py
@@ -128,9 +128,9 @@ class SpiderFoot:
session = self.getSession()
res = session.get(val)
if splitLines:
- return res.content.splitlines()
+ return res.content.decode('utf-8').splitlines()
else:
- return res.content
+ return res.content.decode('utf-8')
except BaseException as e:
if fatal:
self.error("Unable to open option URL, " + val + ": " + str(e))