Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

remove unicode related code

Amr Awad98e7071ff4b1
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
index 3b25d745..6b63067c 100644
--- a/modules/sfp__stor_stdout.py
+++ b/modules/sfp__stor_stdout.py
@@ -75,7 +75,8 @@ class sfp__stor_stdout(SpiderFootPlugin):
if self.opts['_showsource']:
print(('{0:30}\t{1:45}\t{2}\t{3}'.format(event.module, self.opts['_eventtypes'][event.eventType], srcdata, data)))
else:
- print(('{0:30}\t{1:45}\t{2}'.format(event.module, self.opts['_eventtypes'][event.eventType], data)))
+ # ToDo: Is it ok to not find the keys here?
+ print(('{0:30}\t{1:45}\t{2}'.format(event.module, self.opts['_eventtypes'].get(event.eventType, ""), data)))
if self.opts['_format'] == "csv":
print((event.module + d + self.opts['_eventtypes'][event.eventType] + d + srcdata + d + data))
diff --git a/modules/sfp_base64.py b/modules/sfp_base64.py
index 7aa4ea38..c375d01a 100644
--- a/modules/sfp_base64.py
+++ b/modules/sfp_base64.py
@@ -67,7 +67,7 @@ class sfp_base64(SpiderFootPlugin):
return None
self.sf.info("Found Base64 string: " + match)
if type(match) != str:
- string = str(match, 'utf-8', errors='replace')
+ string = str(match)
try:
string += " (" + base64.b64decode(match) + ")"
diff --git a/modules/sfp_crt.py b/modules/sfp_crt.py
index 6df3ca0e..2a3f40da 100644
--- a/modules/sfp_crt.py
+++ b/modules/sfp_crt.py
@@ -62,7 +62,7 @@ class sfp_crt(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
params = {
- 'q': '%.' + eventData.encode('raw_unicode_escape'),
+ 'q': '%.' + eventData,
'output': 'json'
}
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
index 09e3275a..03e43451 100644
--- a/modules/sfp_dnsbrute.py
+++ b/modules/sfp_dnsbrute.py
@@ -148,16 +148,6 @@ class sfp_dnsbrute(SpiderFootPlugin):
return None
self.events[eventDataHash] = True
- # Handle Unicode characters in the name
- try:
- if type(eventData) != str:
- eventData = str(eventData, "utf-8", errors='replace').encode("idna")
- else:
- eventData = eventData.encode("idna")
- except BaseException as e:
- self.sf.debug("IDNA-parsing issue for " + eventData + ": " + str(e))
- return None
-
if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):
if not self.opts['numbersuffix']:
return None
diff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.py
index a4f3f5eb..5dcf47d7 100644
--- a/modules/sfp_dnsraw.py
+++ b/modules/sfp_dnsraw.py
@@ -109,7 +109,7 @@ class sfp_dnsraw(SpiderFootPlugin):
for m in grps:
self.sf.debug("Matched: " + m)
- strdata = str(m, 'utf-8', errors='replace')
+ strdata = str(m)
evt = SpiderFootEvent(recs[rx][1], strdata,
self.__name__, parentEvent)
self.notifyListeners(evt)
@@ -140,7 +140,7 @@ class sfp_dnsraw(SpiderFootPlugin):
evt = SpiderFootEvent(evt_type, domain, self.__name__, parentEvent)
self.notifyListeners(evt)
- strdata = str(x, 'utf-8', errors='replace')
+ strdata = str(x)
evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,
self.__name__, parentEvent)
self.notifyListeners(evt)
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index 922cd2f3..57894cca 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -77,7 +77,7 @@ class sfp_email(SpiderFootPlugin):
self.sf.info("Found e-mail address: " + email)
if type(email) == str:
- mail = unicode(email.strip('.'), 'utf-8', errors='replace')
+ mail = unicode(email.strip('.'))
else:
mail = email.strip('.')
diff --git a/modules/sfp_hostsfilenet.py b/modules/sfp_hostsfilenet.py
index 0510042b..0e50208a 100644
--- a/modules/sfp_hostsfilenet.py
+++ b/modules/sfp_hostsfilenet.py
@@ -89,7 +89,7 @@ class sfp_hostsfilenet(SpiderFootPlugin):
self.sf.cachePut("sfmal_" + cid, data['content'])
if type(data['content']) != str:
- data['content'] = str(data['content'], 'utf-8', errors='replace')
+ data['content'] = str(data['content'])
# Check for the domain and the hostname
if targetType == "domain" and "127.0.0.1\t" + targetDom + "\n" in data['content']:
diff --git a/modules/sfp_names.py b/modules/sfp_names.py
index 4c57907a..5b2ab000 100644
--- a/modules/sfp_names.py
+++ b/modules/sfp_names.py
@@ -79,7 +79,7 @@ class sfp_names(SpiderFootPlugin):
name = " ".join(map(str.capitalize, eventData.split("@")[0].split(".")))
else:
name = " ".join(map(str.capitalize, eventData.split("@")[0].split(".")))
- name = str(name, 'utf-8', errors='replace')
+ name = str(name)
# Names don't have numbers
if re.match("[0-9]", name):
diff --git a/sfdb.py b/sfdb.py
index 64b794d8..09b18c05 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -688,7 +688,7 @@ class SpiderFootDb:
# If sfEvent.data is a dict or list, convert it to a string first, as
# those types do not have a unicode converter.
if type(sfEvent.data) is str:
- storeData = str(sfEvent.data, 'utf-8', errors='replace')
+ storeData = str(sfEvent.data)
else:
try:
storeData = str(sfEvent.data)
diff --git a/sflib.py b/sflib.py
index 041dd0af..9f55da96 100644
--- a/sflib.py
+++ b/sflib.py
@@ -407,8 +407,6 @@ class SpiderFoot:
s = string
if type(string) in [list, dict]:
s = str(string)
- if type(s) == str:
- s = str(s, 'utf-8', errors='replace')
return hashlib.sha256(s.encode('raw_unicode_escape')).hexdigest()
#
@@ -802,10 +800,9 @@ class SpiderFoot:
ret = list()
for addr in res:
if type(addr) == list:
- for host in addr:
- ret.append(str(host, 'utf-8', errors='replace'))
+ ret.extend(addr)
else:
- ret.append(str(addr, 'utf-8', errors='replace'))
+ ret.append(addr)
return list(set(ret))
# Verify input is OK to execute
@@ -920,12 +917,6 @@ class SpiderFoot:
# Return a normalised resolution or None if not resolved.
def resolveHost(self, host):
try:
- # IDNA-encode the hostname in case it contains unicode
- if type(host) != str:
- host = str(host, "utf-8", errors='replace').encode("idna")
- else:
- host = host.encode("idna")
-
addrs = self.normalizeDNS(socket.gethostbyname_ex(host))
if len(addrs) > 0:
return list(set(addrs))
@@ -1218,7 +1209,7 @@ class SpiderFoot:
junk = linkTuple[0]
link = linkTuple[1]
if type(link) != str:
- link = str(link, 'utf-8', errors='replace')
+ link = str(link)
linkl = link.lower()
absLink = None
@@ -1348,7 +1339,7 @@ class SpiderFoot:
if headers is not None:
for k in list(headers.keys()):
if type(headers[k]) != str:
- header[k] = str(headers[k], 'utf-8', errors='replace')
+ header[k] = str(headers[k])
else:
header[k] = headers[k]
@@ -1409,10 +1400,10 @@ class SpiderFoot:
result['headers'] = dict()
for header, value in res.headers.items():
if type(header) != str:
- header = str(header, 'utf-8', errors='replace')
+ header = str(header)
if type(value) != str:
- value = str(value, 'utf-8', errors='replace')
+ value = str(value)
result['headers'][header.lower()] = value
@@ -1441,7 +1432,7 @@ class SpiderFoot:
if dontMangle:
result['content'] = res.content
else:
- result['content'] = str(res.content, 'utf-8', errors='replace')
+ result['content'] = res.content.decode("utf-8")
if fatal:
res.raise_for_status()
except requests.exceptions.HTTPError as h:
@@ -1799,7 +1790,7 @@ class SpiderFootTarget(object):
if typeName in self._validTypes:
self.targetType = typeName
if type(targetValue) != str:
- self.targetValue = str(targetValue, 'utf-8', errors='replace').lower()
+ self.targetValue = str(targetValue).lower()
else:
self.targetValue = targetValue
self.targetAliases = list()
@@ -1937,7 +1928,7 @@ class SpiderFootEvent(object):
sys.exit(-1)
if type(data) != str and data != None:
- self.data = str(data, 'utf-8', errors='replace')
+ self.data = str(data)
else:
self.data = data
diff --git a/sfwebui.py b/sfwebui.py
index 53bcb9b9..4e36a687 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -351,7 +351,7 @@ class SpiderFootWebUi:
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
- return templ.render(id=newId, name=str(scanname, 'utf-8', errors='replace'), docroot=self.docroot,
+ return templ.render(id=newId, name=str(scanname), docroot=self.docroot,
status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
rerunscan.exposed = True
@@ -434,8 +434,8 @@ class SpiderFootWebUi:
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
- scanname=str(scanname, 'utf-8', errors='replace'),
- scantarget=str(scantarget, 'utf-8', errors='replace'))
+ scanname=str(scanname),
+ scantarget=str(scantarget))
clonescan.exposed = True
@@ -528,7 +528,7 @@ class SpiderFootWebUi:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
- return templ.render(id=id, name=str(res[0], 'utf-8', errors='replace'),
+ return templ.render(id=id, name=str(res[0]),
names=list(), ids=list(),
pageid="SCANLIST", docroot=self.docroot)
@@ -541,7 +541,7 @@ class SpiderFootWebUi:
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
- names.append(str(res[0], 'utf-8', errors='replace'))
+ names.append(str(res[0]))
if res is None:
return self.error("Scan ID not found (" + id + ").")
@@ -838,7 +838,7 @@ class SpiderFootWebUi:
if scaninfo is None:
return self.error("Invalid scan ID specified.")
- scanname = str(scaninfo[0], 'utf-8', errors='replace')
+ scanname = str(scaninfo[0])
if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \