Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Bug fixes and consistency improvements.

smicallef8d56cb2ee92d
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.py
index 66ce8919..5bcff5f5 100644
--- a/modules/sfp_binaryedge.py
+++ b/modules/sfp_binaryedge.py
@@ -246,7 +246,7 @@ class sfp_binaryedge(SpiderFootPlugin):
self.sf.debug("Found compromised account results in BinaryEdge.io")
res = rec["events"]
for rec in res:
- e = SpiderFootEvent(evtType, rec, self.__name__, event)
+ e = SpiderFootEvent(evtType, eventData + " [" + rec + "]", self.__name__, event)
self.notifyListeners(e)
if eventName == "DOMAIN_NAME":
diff --git a/modules/sfp_censys.py b/modules/sfp_censys.py
index a419f7f5..04fba76f 100644
--- a/modules/sfp_censys.py
+++ b/modules/sfp_censys.py
@@ -142,8 +142,6 @@ class sfp_censys(SpiderFootPlugin):
if rec is None:
continue
- self.sf.debug("Censys raw data: " + str(rec))
-
if 'error' in rec:
if rec['error_type'] == "unknown":
self.sf.debug("Censys returned no data for " + addr)
@@ -158,7 +156,7 @@ class sfp_censys(SpiderFootPlugin):
try:
# Date format: 2016-12-24T07:25:35+00:00'
- created_dt = datetime.strptime(rec.get('updated_at'), '%Y-%m-%dT%H:%M:%S+00:00')
+ created_dt = datetime.strptime(rec.get('updated_at', "1970-01-01T00:00:00+00:00"), '%Y-%m-%dT%H:%M:%S+00:00')
created_ts = int(time.mktime(created_dt.timetuple()))
age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])
diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.py
index 757aa5ae..585ba8c6 100644
--- a/modules/sfp_citadel.py
+++ b/modules/sfp_citadel.py
@@ -86,6 +86,9 @@ class sfp_citadel(SpiderFootPlugin):
eventData = event.data
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if self.errorState:
+ return None
# Don't look up stuff twice
if eventData in self.results:
@@ -104,6 +107,9 @@ class sfp_citadel(SpiderFootPlugin):
if error == 'true':
self.sf.error("Error encountered processing {}: {}".format( eventData, message ), False)
+ if "MISSING API" in message:
+ self.errorState = True
+ return None
return None
if not message:
diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.py
index d85b0470..b8fe631e 100644
--- a/modules/sfp_commoncrawl.py
+++ b/modules/sfp_commoncrawl.py
@@ -122,6 +122,9 @@ class sfp_commoncrawl(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+ if self.errorState:
+ return None
+
if eventData in self.results:
return None
else:
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index a86dcc47..cee213a5 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -62,6 +62,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
return target
for host in ret:
+ self.sf.debug("Found an alias: " + host)
if self.sf.validIP(host):
target.setAlias(host, "IP_ADDRESS")
else:
diff --git a/modules/sfp_github.py b/modules/sfp_github.py
index 51cfc97f..25397689 100644
--- a/modules/sfp_github.py
+++ b/modules/sfp_github.py
@@ -81,7 +81,7 @@ class sfp_github(SpiderFootPlugin):
if eventName == "SOCIAL_MEDIA":
try:
network = eventData.split(": ")[0]
- url = eventData.split(": ")[1]
+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index b66b5ec4..e96e7438 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -160,7 +160,7 @@ class sfp_gravatar(SpiderFootPlugin):
url = account.get('url')
platform = account.get('shortname')
if platform is not None and url is not None:
- t = platform.capitalize() + ": " + url
+ t = platform.capitalize() + ": <SFURL>" + url + "</SFURL>"
evt = SpiderFootEvent("SOCIAL_MEDIA", t, self.__name__, event)
self.notifyListeners(evt)
diff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.py
index be7a71e5..3abc7e52 100644
--- a/modules/sfp_instagram.py
+++ b/modules/sfp_instagram.py
@@ -77,7 +77,7 @@ class sfp_instagram(SpiderFootPlugin):
# Parse profile URL
try:
network = eventData.split(": ")[0]
- url = eventData.split(": ")[1]
+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.py
index 98686992..90a65d30 100644
--- a/modules/sfp_myspace.py
+++ b/modules/sfp_myspace.py
@@ -91,7 +91,8 @@ class sfp_myspace(SpiderFootPlugin):
return None
name = matches[0]
- e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + "https://myspace.com/" + name,
+ e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + \
+ "<SFURL>https://myspace.com/" + name + "</SFURL>",
self.__name__, event)
self.notifyListeners(e)
@@ -99,7 +100,7 @@ class sfp_myspace(SpiderFootPlugin):
if eventName == "SOCIAL_MEDIA":
try:
network = eventData.split(": ")[0]
- url = eventData.split(": ")[1]
+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.py
index ff44c300..e46a7c5d 100644
--- a/modules/sfp_neutrinoapi.py
+++ b/modules/sfp_neutrinoapi.py
@@ -107,6 +107,9 @@ class sfp_neutrinoapi(SpiderFootPlugin):
if res['code'] == "400":
if data.get('api-error-msg'):
self.sf.error("Error: " + data.get('api-error-msg'), False)
+ if "EXCEED" in data.get('api-error-msg'):
+ self.errorState = True
+ return None
else:
self.sf.error("Error: HTTP 400", False)
return None
diff --git a/modules/sfp_openstreetmap.py b/modules/sfp_openstreetmap.py
index 984b1248..f4518d10 100644
--- a/modules/sfp_openstreetmap.py
+++ b/modules/sfp_openstreetmap.py
@@ -92,11 +92,13 @@ class sfp_openstreetmap(SpiderFootPlugin):
self.sf.debug("Skipping PO BOX address")
return None
+ rx1 = re.compile(r'^(c/o|care of|attn:|attention:)\s+[0-9a-z\s\.]', flags=re.IGNORECASE)
# Remove address prefixes for delivery instructions
- address = re.sub(r'^(c/o|care of|attn:|attention:)\s+[0-9a-z\s\.],', r'', address, flags=re.IGNORECASE)
+ address = re.sub(rx1, r'', address)
+ rx2 = re.compile(r'^(Level|Floor|Suite|Room)\s+[0-9a-z]+,', flags=re.IGNORECASE)
# Remove address prefixes known to return no results (floor, level, suite, etc).
- address = re.sub(r'^(Level|Floor|Suite|Room)\s+[0-9a-z]+,', r'', address, flags=re.IGNORECASE)
+ address = re.sub(rx2, r'', address)
# Search for address
data = self.query(eventData)
diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.py
index 27edbefa..0c598d27 100644
--- a/modules/sfp_pgp.py
+++ b/modules/sfp_pgp.py
@@ -74,7 +74,7 @@ class sfp_pgp(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
# Get e-mail addresses on this domain
- if eventName == "DOMAIN_NAME":
+ if eventName in ["DOMAIN_NAME", "INTERNET_NAME"]:
res = self.sf.fetchUrl(self.opts['keyserver_search1'] + eventData,
timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
diff --git a/modules/sfp_pulsedive.py b/modules/sfp_pulsedive.py
index ece64ac6..d7b23f82 100644
--- a/modules/sfp_pulsedive.py
+++ b/modules/sfp_pulsedive.py
@@ -93,7 +93,6 @@ class sfp_pulsedive(SpiderFootPlugin):
try:
info = json.loads(res['content'])
except Exception as e:
- print(str(res['content']))
self.sf.error("Error processing JSON response from Pulsedive.", False)
return None
diff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.py
index c6ce3a16..b31f5e2c 100644
--- a/modules/sfp_slideshare.py
+++ b/modules/sfp_slideshare.py
@@ -63,7 +63,7 @@ class sfp_slideshare(SpiderFootPlugin):
# Retrieve profile
try:
network = eventData.split(": ")[0]
- url = eventData.split(": ")[1]
+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.py
index 9c63acad..2554f920 100644
--- a/modules/sfp_twitter.py
+++ b/modules/sfp_twitter.py
@@ -59,7 +59,7 @@ class sfp_twitter(SpiderFootPlugin):
# Retrieve profile
try:
network = eventData.split(": ")[0]
- url = eventData.split(": ")[1]
+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)