SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
More unicode fixes.
Steve Micallef786eff4733bf
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.pyindex 7562bde9..431aa1eb 100644--- a/modules/sfp_dnsbrute.py+++ b/modules/sfp_dnsbrute.py@@ -90,23 +90,6 @@ class sfp_dnsbrute(SpiderFootPlugin):def producedEvents(self):return ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS"]- # Resolve a host- def resolveHost(self, hostname):- if hostname in self.resolveCache:- self.sf.debug("Returning cached result for " + hostname + " (" +- str(self.resolveCache[hostname]) + ")")- return self.resolveCache[hostname]-- try:- hostname = hostname.encode("idna")- addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))- self.resolveCache[hostname] = addrs- self.sf.debug("Resolved " + hostname + " to: " + str(addrs))- return addrs- except BaseException as e:- self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")- return list()-def tryHost(self, name):resolver = dns.resolver.Resolver()resolver.timeout = 2@@ -114,6 +97,12 @@ class sfp_dnsbrute(SpiderFootPlugin):resolver.search = list()try:+ # IDNA-encode the hostname in case it contains unicode+ if type(name) != unicode:+ name = unicode(name, "utf-8", errors='replace').encode("idna")+ else:+ name = name.encode("idna")+addrs = resolver.query(name)self.hostResults[name] = Trueexcept BaseException as e:@@ -171,7 +160,10 @@ class sfp_dnsbrute(SpiderFootPlugin):self.events[eventDataHash] = True# Handle Unicode characters in the name- eventData = eventData.encode("idna")+ if type(eventData) != unicode:+ eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")+ else:+ eventData = eventData.encode("idna")if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):if not self.opts['numbersuffix']:diff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.pyindex 6c4ba82b..40d208ba 100644--- a/modules/sfp_dnsneighbor.py+++ b/modules/sfp_dnsneighbor.py@@ -169,7 +169,12 @@ class sfp_dnsneighbor(SpiderFootPlugin):return self.resolveCache[hostname]try:- hostname = hostname.encode("idna")+ # IDNA-encode the hostname in case it contains unicode+ if type(hostname) != unicode:+ hostname = unicode(hostname, "utf-8", errors='replace').encode("idna")+ else:+ hostname = hostname.encode("idna")+addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))self.resolveCache[hostname] = addrsself.sf.debug("Resolved " + hostname + " to: " + str(addrs))diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.pyindex 9b6c2d57..caf195ac 100644--- a/modules/sfp_dnsresolve.py+++ b/modules/sfp_dnsresolve.py@@ -271,7 +271,10 @@ class sfp_dnsresolve(SpiderFootPlugin):try:# IDNA-encode the hostname in case it contains unicode- hostname = hostname.encode("idna")+ if type(hostname) != unicode:+ hostname = unicode(hostname, "utf-8", errors='replace').encode("idna")+ else:+ hostname = hostname.encode("idna")addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))self.resolveCache[hostname] = addrsself.sf.debug("Resolved " + hostname + " to: " + str(addrs))diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 17bd59bd..c5aa449a 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -239,7 +239,7 @@ class sfp_spider(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)- if eventData in self.urlEvents.keys():+ if eventData in self.urlEvents:self.sf.debug("Ignoring " + eventData + " as already spidered or is being spidered.")return Noneelse:@@ -252,7 +252,8 @@ class sfp_spider(SpiderFootPlugin):# Determine where to start spidering from if it's a INTERNET_NAME eventif eventName == "INTERNET_NAME":for prefix in self.opts['start']:- res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],+ url = prefix + eventData+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])if res['content'] is not None:spiderTarget = prefix + eventDatadiff --git a/modules/sfp_sublist3r.py b/modules/sfp_sublist3r.pyindex a18fe895..109fd501 100644--- a/modules/sfp_sublist3r.py+++ b/modules/sfp_sublist3r.py@@ -51,7 +51,12 @@ class sfp_sublist3r(SpiderFootPlugin):# Verify a host resolvesdef resolveHost(self, host):try:- host = host.encode("idna")+ # IDNA-encode the hostname in case it contains unicode+ if type(host) != unicode:+ host = unicode(host, "utf-8", errors='replace').encode("idna")+ else:+ host = host.encode("idna")+addrs = socket.gethostbyname_ex(host)except BaseException as e:self.sf.debug("Unable to resolve " + host + ": " + str(e))diff --git a/modules/sfp_virustotal.py b/modules/sfp_virustotal.pyindex 36099521..d2c1cdfc 100644--- a/modules/sfp_virustotal.py+++ b/modules/sfp_virustotal.py@@ -80,7 +80,12 @@ class sfp_virustotal(SpiderFootPlugin):# Verify a host resolvesdef resolveHost(self, host):try:- host = host.encode("idna")+ # IDNA-encode the hostname in case it contains unicode+ if type(host) != unicode:+ host = unicode(host, "utf-8", errors='replace').encode("idna")+ else:+ host = host.encode("idna")+addrs = socket.gethostbyname_ex(host)except BaseException as e:self.sf.debug("Unable to resolve " + host + ": " + str(e))diff --git a/sflib.py b/sflib.pyindex 2b7e7440..13d71073 100644--- a/sflib.py+++ b/sflib.py@@ -994,7 +994,7 @@ class SpiderFoot:url = unicode(url, 'utf-8', errors='replace')# Convert any unicode chars in the URL- url = self.urlEncodeUnicode(url)+ url = str(self.urlEncodeUnicode(url))try:header = dict()@@ -1492,13 +1492,14 @@ class SpiderFootPlugin(object):# within the same execution context of this thread, not on their own.def notifyListeners(self, sfEvent):eventName = sfEvent.eventType++ # Convert strings to unicodeif type(sfEvent.data) == str:- eventData = unicode(sfEvent.data, 'utf-8', errors='replace')- else:- eventData = sfEvent.data+ sfEvent.data = unicode(sfEvent.data, 'utf-8', errors='replace')+storeOnly = False # Under some conditions, only store and don't notify- if eventData is None or (type(eventData) is unicode and len(eventData) == 0):+ if sfEvent.data is None or (type(sfEvent.data) is unicode and len(sfEvent.data) == 0):#print "No data to send for " + eventName + " to " + listener.__module__return None@@ -1552,7 +1553,7 @@ class SpiderFootPlugin(object):try:listener.handleEvent(sfEvent)except BaseException as e:- self.sf.error("Module encountered an error: " + str(e))+ print "Module encountered an error: " + str(e)# For modules to use to check for when they should give back controldef checkForStop(self):@@ -1604,7 +1605,7 @@ class SpiderFootTarget(object):def __init__(self, targetValue, typeName):if typeName in self._validTypes:self.targetType = typeName- self.targetValue = targetValue.lower()+ self.targetValue = unicode(targetValue.lower(), 'utf-8', errors='replace')self.targetAliases = list()else:print "Internal Error: Invalid target type."