SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
remove unicode related code
Amr Awad98e7071ff4b1
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pyindex 3b25d745..6b63067c 100644--- a/modules/sfp__stor_stdout.py+++ b/modules/sfp__stor_stdout.py@@ -75,7 +75,8 @@ class sfp__stor_stdout(SpiderFootPlugin):if self.opts['_showsource']:print(('{0:30}\t{1:45}\t{2}\t{3}'.format(event.module, self.opts['_eventtypes'][event.eventType], srcdata, data)))else:- print(('{0:30}\t{1:45}\t{2}'.format(event.module, self.opts['_eventtypes'][event.eventType], data)))+ # ToDo: Is it ok to not find the keys here?+ print(('{0:30}\t{1:45}\t{2}'.format(event.module, self.opts['_eventtypes'].get(event.eventType, ""), data)))if self.opts['_format'] == "csv":print((event.module + d + self.opts['_eventtypes'][event.eventType] + d + srcdata + d + data))diff --git a/modules/sfp_base64.py b/modules/sfp_base64.pyindex 7aa4ea38..c375d01a 100644--- a/modules/sfp_base64.py+++ b/modules/sfp_base64.py@@ -67,7 +67,7 @@ class sfp_base64(SpiderFootPlugin):return Noneself.sf.info("Found Base64 string: " + match)if type(match) != str:- string = str(match, 'utf-8', errors='replace')+ string = str(match)try:string += " (" + base64.b64decode(match) + ")"diff --git a/modules/sfp_crt.py b/modules/sfp_crt.pyindex 6df3ca0e..2a3f40da 100644--- a/modules/sfp_crt.py+++ b/modules/sfp_crt.py@@ -62,7 +62,7 @@ class sfp_crt(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)params = {- 'q': '%.' + eventData.encode('raw_unicode_escape'),+ 'q': '%.' + eventData,'output': 'json'}diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pyindex 09e3275a..03e43451 100644--- a/modules/sfp_dnsbrute.py+++ b/modules/sfp_dnsbrute.py@@ -148,16 +148,6 @@ class sfp_dnsbrute(SpiderFootPlugin):return Noneself.events[eventDataHash] = True- # Handle Unicode characters in the name- try:- if type(eventData) != str:- eventData = str(eventData, "utf-8", errors='replace').encode("idna")- else:- eventData = eventData.encode("idna")- except BaseException as e:- self.sf.debug("IDNA-parsing issue for " + eventData + ": " + str(e))- return None-if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):if not self.opts['numbersuffix']:return Nonediff --git a/modules/sfp_dnsraw.py b/modules/sfp_dnsraw.pyindex a4f3f5eb..5dcf47d7 100644--- a/modules/sfp_dnsraw.py+++ b/modules/sfp_dnsraw.py@@ -109,7 +109,7 @@ class sfp_dnsraw(SpiderFootPlugin):for m in grps:self.sf.debug("Matched: " + m)- strdata = str(m, 'utf-8', errors='replace')+ strdata = str(m)evt = SpiderFootEvent(recs[rx][1], strdata,self.__name__, parentEvent)self.notifyListeners(evt)@@ -140,7 +140,7 @@ class sfp_dnsraw(SpiderFootPlugin):evt = SpiderFootEvent(evt_type, domain, self.__name__, parentEvent)self.notifyListeners(evt)- strdata = str(x, 'utf-8', errors='replace')+ strdata = str(x)evt = SpiderFootEvent("RAW_DNS_RECORDS", strdata,self.__name__, parentEvent)self.notifyListeners(evt)diff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex 922cd2f3..57894cca 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -77,7 +77,7 @@ class sfp_email(SpiderFootPlugin):self.sf.info("Found e-mail address: " + email)if type(email) == str:- mail = unicode(email.strip('.'), 'utf-8', errors='replace')+ mail = unicode(email.strip('.'))else:mail = email.strip('.')diff --git a/modules/sfp_hostsfilenet.py b/modules/sfp_hostsfilenet.pyindex 0510042b..0e50208a 100644--- a/modules/sfp_hostsfilenet.py+++ b/modules/sfp_hostsfilenet.py@@ -89,7 +89,7 @@ class sfp_hostsfilenet(SpiderFootPlugin):self.sf.cachePut("sfmal_" + cid, data['content'])if type(data['content']) != str:- data['content'] = str(data['content'], 'utf-8', errors='replace')+ data['content'] = str(data['content'])# Check for the domain and the hostnameif targetType == "domain" and "127.0.0.1\t" + targetDom + "\n" in data['content']:diff --git a/modules/sfp_names.py b/modules/sfp_names.pyindex 4c57907a..5b2ab000 100644--- a/modules/sfp_names.py+++ b/modules/sfp_names.py@@ -79,7 +79,7 @@ class sfp_names(SpiderFootPlugin):name = " ".join(map(str.capitalize, eventData.split("@")[0].split(".")))else:name = " ".join(map(str.capitalize, eventData.split("@")[0].split(".")))- name = str(name, 'utf-8', errors='replace')+ name = str(name)# Names don't have numbersif re.match("[0-9]", name):diff --git a/sfdb.py b/sfdb.pyindex 64b794d8..09b18c05 100644--- a/sfdb.py+++ b/sfdb.py@@ -688,7 +688,7 @@ class SpiderFootDb:# If sfEvent.data is a dict or list, convert it to a string first, as# those types do not have a unicode converter.if type(sfEvent.data) is str:- storeData = str(sfEvent.data, 'utf-8', errors='replace')+ storeData = str(sfEvent.data)else:try:storeData = str(sfEvent.data)diff --git a/sflib.py b/sflib.pyindex 041dd0af..9f55da96 100644--- a/sflib.py+++ b/sflib.py@@ -407,8 +407,6 @@ class SpiderFoot:s = stringif type(string) in [list, dict]:s = str(string)- if type(s) == str:- s = str(s, 'utf-8', errors='replace')return hashlib.sha256(s.encode('raw_unicode_escape')).hexdigest()#@@ -802,10 +800,9 @@ class SpiderFoot:ret = list()for addr in res:if type(addr) == list:- for host in addr:- ret.append(str(host, 'utf-8', errors='replace'))+ ret.extend(addr)else:- ret.append(str(addr, 'utf-8', errors='replace'))+ ret.append(addr)return list(set(ret))# Verify input is OK to execute@@ -920,12 +917,6 @@ class SpiderFoot:# Return a normalised resolution or None if not resolved.def resolveHost(self, host):try:- # IDNA-encode the hostname in case it contains unicode- if type(host) != str:- host = str(host, "utf-8", errors='replace').encode("idna")- else:- host = host.encode("idna")-addrs = self.normalizeDNS(socket.gethostbyname_ex(host))if len(addrs) > 0:return list(set(addrs))@@ -1218,7 +1209,7 @@ class SpiderFoot:junk = linkTuple[0]link = linkTuple[1]if type(link) != str:- link = str(link, 'utf-8', errors='replace')+ link = str(link)linkl = link.lower()absLink = None@@ -1348,7 +1339,7 @@ class SpiderFoot:if headers is not None:for k in list(headers.keys()):if type(headers[k]) != str:- header[k] = str(headers[k], 'utf-8', errors='replace')+ header[k] = str(headers[k])else:header[k] = headers[k]@@ -1409,10 +1400,10 @@ class SpiderFoot:result['headers'] = dict()for header, value in res.headers.items():if type(header) != str:- header = str(header, 'utf-8', errors='replace')+ header = str(header)if type(value) != str:- value = str(value, 'utf-8', errors='replace')+ value = str(value)result['headers'][header.lower()] = value@@ -1441,7 +1432,7 @@ class SpiderFoot:if dontMangle:result['content'] = res.contentelse:- result['content'] = str(res.content, 'utf-8', errors='replace')+ result['content'] = res.content.decode("utf-8")if fatal:res.raise_for_status()except requests.exceptions.HTTPError as h:@@ -1799,7 +1790,7 @@ class SpiderFootTarget(object):if typeName in self._validTypes:self.targetType = typeNameif type(targetValue) != str:- self.targetValue = str(targetValue, 'utf-8', errors='replace').lower()+ self.targetValue = str(targetValue).lower()else:self.targetValue = targetValueself.targetAliases = list()@@ -1937,7 +1928,7 @@ class SpiderFootEvent(object):sys.exit(-1)if type(data) != str and data != None:- self.data = str(data, 'utf-8', errors='replace')+ self.data = str(data)else:self.data = datadiff --git a/sfwebui.py b/sfwebui.pyindex 53bcb9b9..4e36a687 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -351,7 +351,7 @@ class SpiderFootWebUi:time.sleep(1)templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)- return templ.render(id=newId, name=str(scanname, 'utf-8', errors='replace'), docroot=self.docroot,+ return templ.render(id=newId, name=str(scanname), docroot=self.docroot,status=globalScanStatus.getStatus(newId), pageid="SCANLIST")rerunscan.exposed = True@@ -434,8 +434,8 @@ class SpiderFootWebUi:templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,modules=self.config['__modules__'], selectedmods=modlist,- scanname=str(scanname, 'utf-8', errors='replace'),- scantarget=str(scantarget, 'utf-8', errors='replace'))+ scanname=str(scanname),+ scantarget=str(scantarget))clonescan.exposed = True@@ -528,7 +528,7 @@ class SpiderFootWebUi:return json.dumps(["SUCCESS", ""])else:templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)- return templ.render(id=id, name=str(res[0], 'utf-8', errors='replace'),+ return templ.render(id=id, name=str(res[0]),names=list(), ids=list(),pageid="SCANLIST", docroot=self.docroot)@@ -541,7 +541,7 @@ class SpiderFootWebUi:for id in ids.split(','):res = dbh.scanInstanceGet(id)- names.append(str(res[0], 'utf-8', errors='replace'))+ names.append(str(res[0]))if res is None:return self.error("Scan ID not found (" + id + ").")@@ -838,7 +838,7 @@ class SpiderFootWebUi:if scaninfo is None:return self.error("Invalid scan ID specified.")- scanname = str(scaninfo[0], 'utf-8', errors='replace')+ scanname = str(scaninfo[0])if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED":error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \