Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

More unicode fixes.

Steve Micallef786eff4733bf
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
index 7562bde9..431aa1eb 100644
--- a/modules/sfp_dnsbrute.py
+++ b/modules/sfp_dnsbrute.py
@@ -90,23 +90,6 @@ class sfp_dnsbrute(SpiderFootPlugin):
def producedEvents(self):
return ["IP_ADDRESS", "INTERNET_NAME", "IPV6_ADDRESS"]
- # Resolve a host
- def resolveHost(self, hostname):
- if hostname in self.resolveCache:
- self.sf.debug("Returning cached result for " + hostname + " (" +
- str(self.resolveCache[hostname]) + ")")
- return self.resolveCache[hostname]
-
- try:
- hostname = hostname.encode("idna")
- addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
- self.resolveCache[hostname] = addrs
- self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
- return addrs
- except BaseException as e:
- self.sf.debug("Unable to resolve " + hostname + " (" + str(e) + ")")
- return list()
-
def tryHost(self, name):
resolver = dns.resolver.Resolver()
resolver.timeout = 2
@@ -114,6 +97,12 @@ class sfp_dnsbrute(SpiderFootPlugin):
resolver.search = list()
try:
+ # IDNA-encode the hostname in case it contains unicode
+ if type(name) != unicode:
+ name = unicode(name, "utf-8", errors='replace').encode("idna")
+ else:
+ name = name.encode("idna")
+
addrs = resolver.query(name)
self.hostResults[name] = True
except BaseException as e:
@@ -171,7 +160,10 @@ class sfp_dnsbrute(SpiderFootPlugin):
self.events[eventDataHash] = True
# Handle Unicode characters in the name
- eventData = eventData.encode("idna")
+ if type(eventData) != unicode:
+ eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")
+ else:
+ eventData = eventData.encode("idna")
if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):
if not self.opts['numbersuffix']:
diff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.py
index 6c4ba82b..40d208ba 100644
--- a/modules/sfp_dnsneighbor.py
+++ b/modules/sfp_dnsneighbor.py
@@ -169,7 +169,12 @@ class sfp_dnsneighbor(SpiderFootPlugin):
return self.resolveCache[hostname]
try:
- hostname = hostname.encode("idna")
+ # IDNA-encode the hostname in case it contains unicode
+ if type(hostname) != unicode:
+ hostname = unicode(hostname, "utf-8", errors='replace').encode("idna")
+ else:
+ hostname = hostname.encode("idna")
+
addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
self.resolveCache[hostname] = addrs
self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index 9b6c2d57..caf195ac 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -271,7 +271,10 @@ class sfp_dnsresolve(SpiderFootPlugin):
try:
# IDNA-encode the hostname in case it contains unicode
- hostname = hostname.encode("idna")
+ if type(hostname) != unicode:
+ hostname = unicode(hostname, "utf-8", errors='replace').encode("idna")
+ else:
+ hostname = hostname.encode("idna")
addrs = self.sf.normalizeDNS(socket.gethostbyname_ex(hostname))
self.resolveCache[hostname] = addrs
self.sf.debug("Resolved " + hostname + " to: " + str(addrs))
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 17bd59bd..c5aa449a 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -239,7 +239,7 @@ class sfp_spider(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
- if eventData in self.urlEvents.keys():
+ if eventData in self.urlEvents:
self.sf.debug("Ignoring " + eventData + " as already spidered or is being spidered.")
return None
else:
@@ -252,7 +252,8 @@ class sfp_spider(SpiderFootPlugin):
# Determine where to start spidering from if it's a INTERNET_NAME event
if eventName == "INTERNET_NAME":
for prefix in self.opts['start']:
- res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],
+ url = prefix + eventData
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
if res['content'] is not None:
spiderTarget = prefix + eventData
diff --git a/modules/sfp_sublist3r.py b/modules/sfp_sublist3r.py
index a18fe895..109fd501 100644
--- a/modules/sfp_sublist3r.py
+++ b/modules/sfp_sublist3r.py
@@ -51,7 +51,12 @@ class sfp_sublist3r(SpiderFootPlugin):
# Verify a host resolves
def resolveHost(self, host):
try:
- host = host.encode("idna")
+ # IDNA-encode the hostname in case it contains unicode
+ if type(host) != unicode:
+ host = unicode(host, "utf-8", errors='replace').encode("idna")
+ else:
+ host = host.encode("idna")
+
addrs = socket.gethostbyname_ex(host)
except BaseException as e:
self.sf.debug("Unable to resolve " + host + ": " + str(e))
diff --git a/modules/sfp_virustotal.py b/modules/sfp_virustotal.py
index 36099521..d2c1cdfc 100644
--- a/modules/sfp_virustotal.py
+++ b/modules/sfp_virustotal.py
@@ -80,7 +80,12 @@ class sfp_virustotal(SpiderFootPlugin):
# Verify a host resolves
def resolveHost(self, host):
try:
- host = host.encode("idna")
+ # IDNA-encode the hostname in case it contains unicode
+ if type(host) != unicode:
+ host = unicode(host, "utf-8", errors='replace').encode("idna")
+ else:
+ host = host.encode("idna")
+
addrs = socket.gethostbyname_ex(host)
except BaseException as e:
self.sf.debug("Unable to resolve " + host + ": " + str(e))
diff --git a/sflib.py b/sflib.py
index 2b7e7440..13d71073 100644
--- a/sflib.py
+++ b/sflib.py
@@ -994,7 +994,7 @@ class SpiderFoot:
url = unicode(url, 'utf-8', errors='replace')
# Convert any unicode chars in the URL
- url = self.urlEncodeUnicode(url)
+ url = str(self.urlEncodeUnicode(url))
try:
header = dict()
@@ -1492,13 +1492,14 @@ class SpiderFootPlugin(object):
# within the same execution context of this thread, not on their own.
def notifyListeners(self, sfEvent):
eventName = sfEvent.eventType
+
+ # Convert strings to unicode
if type(sfEvent.data) == str:
- eventData = unicode(sfEvent.data, 'utf-8', errors='replace')
- else:
- eventData = sfEvent.data
+ sfEvent.data = unicode(sfEvent.data, 'utf-8', errors='replace')
+
storeOnly = False # Under some conditions, only store and don't notify
- if eventData is None or (type(eventData) is unicode and len(eventData) == 0):
+ if sfEvent.data is None or (type(sfEvent.data) is unicode and len(sfEvent.data) == 0):
#print "No data to send for " + eventName + " to " + listener.__module__
return None
@@ -1552,7 +1553,7 @@ class SpiderFootPlugin(object):
try:
listener.handleEvent(sfEvent)
except BaseException as e:
- self.sf.error("Module encountered an error: " + str(e))
+ print "Module encountered an error: " + str(e)
# For modules to use to check for when they should give back control
def checkForStop(self):
@@ -1604,7 +1605,7 @@ class SpiderFootTarget(object):
def __init__(self, targetValue, typeName):
if typeName in self._validTypes:
self.targetType = typeName
- self.targetValue = targetValue.lower()
+ self.targetValue = unicode(targetValue.lower(), 'utf-8', errors='replace')
self.targetAliases = list()
else:
print "Internal Error: Invalid target type."