Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Bug fixes.

Stevedd48d5ede6c1
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index 66e7b650..5820264a 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -135,7 +135,11 @@ class sfp_dnsresolve(SpiderFootPlugin):
# Search for IPs/hosts in raw data, but obviously nothing this module
# already produces, as those things are already entities, not raw data.
if eventName not in self.producedEvents():
- data = urllib2.unquote(eventData)
+ if type(eventData) in [dict, list]:
+ eventDataStr = str(eventData)
+ else:
+ eventDataStr = eventData
+ data = urllib2.unquote(eventDataStr)
for name in self.getTarget().getNames():
if self.checkForStop():
return None
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 1586c811..027d9e24 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -166,10 +166,11 @@ class sfp_spider(SpiderFootPlugin):
continue
# If we are respecting robots.txt, filter those out too
- checkRobots = lambda blocked: str.lower(blocked) in link.lower() or blocked == '*'
- if self.opts['robotsonly'] and filter(checkRobots, self.robotsRules[linkBase]):
- #self.sf.debug("Ignoring page found in robots.txt: " + link)
- continue
+ if linkBase in self.robotsRules and self.opts['robotsonly']:
+ checkRobots = lambda blocked: type(blocked).lower(blocked) in link.lower() or blocked == '*'
+ if filter(checkRobots, self.robotsRules[linkBase]):
+ #self.sf.debug("Ignoring page found in robots.txt: " + link)
+ continue
# All tests passed, add link to be spidered
self.sf.debug("Adding URL for spidering: " + link)
diff --git a/sflib.py b/sflib.py
index 16f3b879..4be41666 100644
--- a/sflib.py
+++ b/sflib.py
@@ -369,6 +369,8 @@ class SpiderFoot:
return os.path.dirname(unicode(__file__, sys.getfilesystemencoding()))
def hashstring(self, string):
+ if type(string) in [dict, list]:
+ string = str(string)
s = string.encode('raw_unicode_escape')
return hashlib.sha256(s).hexdigest()