SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Bug fixes and consistency improvements.
smicallef8d56cb2ee92d
diff --git a/modules/sfp_binaryedge.py b/modules/sfp_binaryedge.pyindex 66ce8919..5bcff5f5 100644--- a/modules/sfp_binaryedge.py+++ b/modules/sfp_binaryedge.py@@ -246,7 +246,7 @@ class sfp_binaryedge(SpiderFootPlugin):self.sf.debug("Found compromised account results in BinaryEdge.io")res = rec["events"]for rec in res:- e = SpiderFootEvent(evtType, rec, self.__name__, event)+ e = SpiderFootEvent(evtType, eventData + " [" + rec + "]", self.__name__, event)self.notifyListeners(e)if eventName == "DOMAIN_NAME":diff --git a/modules/sfp_censys.py b/modules/sfp_censys.pyindex a419f7f5..04fba76f 100644--- a/modules/sfp_censys.py+++ b/modules/sfp_censys.py@@ -142,8 +142,6 @@ class sfp_censys(SpiderFootPlugin):if rec is None:continue- self.sf.debug("Censys raw data: " + str(rec))-if 'error' in rec:if rec['error_type'] == "unknown":self.sf.debug("Censys returned no data for " + addr)@@ -158,7 +156,7 @@ class sfp_censys(SpiderFootPlugin):try:# Date format: 2016-12-24T07:25:35+00:00'- created_dt = datetime.strptime(rec.get('updated_at'), '%Y-%m-%dT%H:%M:%S+00:00')+ created_dt = datetime.strptime(rec.get('updated_at', "1970-01-01T00:00:00+00:00"), '%Y-%m-%dT%H:%M:%S+00:00')created_ts = int(time.mktime(created_dt.timetuple()))age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.pyindex 757aa5ae..585ba8c6 100644--- a/modules/sfp_citadel.py+++ b/modules/sfp_citadel.py@@ -86,6 +86,9 @@ class sfp_citadel(SpiderFootPlugin):eventData = event.dataself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.errorState:+ return None# Don't look up stuff twiceif eventData in self.results:@@ -104,6 +107,9 @@ class sfp_citadel(SpiderFootPlugin):if error == 'true':self.sf.error("Error encountered processing {}: {}".format( eventData, message ), False)+ if "MISSING API" in message:+ self.errorState = True+ return Nonereturn Noneif not message:diff --git a/modules/sfp_commoncrawl.py b/modules/sfp_commoncrawl.pyindex d85b0470..b8fe631e 100644--- a/modules/sfp_commoncrawl.py+++ b/modules/sfp_commoncrawl.py@@ -122,6 +122,9 @@ class sfp_commoncrawl(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)+ if self.errorState:+ return None+if eventData in self.results:return Noneelse:diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.pyindex a86dcc47..cee213a5 100644--- a/modules/sfp_dnsresolve.py+++ b/modules/sfp_dnsresolve.py@@ -62,6 +62,7 @@ class sfp_dnsresolve(SpiderFootPlugin):return targetfor host in ret:+ self.sf.debug("Found an alias: " + host)if self.sf.validIP(host):target.setAlias(host, "IP_ADDRESS")else:diff --git a/modules/sfp_github.py b/modules/sfp_github.pyindex 51cfc97f..25397689 100644--- a/modules/sfp_github.py+++ b/modules/sfp_github.py@@ -81,7 +81,7 @@ class sfp_github(SpiderFootPlugin):if eventName == "SOCIAL_MEDIA":try:network = eventData.split(": ")[0]- url = eventData.split(": ")[1]+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex b66b5ec4..e96e7438 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -160,7 +160,7 @@ class sfp_gravatar(SpiderFootPlugin):url = account.get('url')platform = account.get('shortname')if platform is not None and url is not None:- t = platform.capitalize() + ": " + url+ t = platform.capitalize() + ": <SFURL>" + url + "</SFURL>"evt = SpiderFootEvent("SOCIAL_MEDIA", t, self.__name__, event)self.notifyListeners(evt)diff --git a/modules/sfp_instagram.py b/modules/sfp_instagram.pyindex be7a71e5..3abc7e52 100644--- a/modules/sfp_instagram.py+++ b/modules/sfp_instagram.py@@ -77,7 +77,7 @@ class sfp_instagram(SpiderFootPlugin):# Parse profile URLtry:network = eventData.split(": ")[0]- url = eventData.split(": ")[1]+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.pyindex 98686992..90a65d30 100644--- a/modules/sfp_myspace.py+++ b/modules/sfp_myspace.py@@ -91,7 +91,8 @@ class sfp_myspace(SpiderFootPlugin):return Nonename = matches[0]- e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + "https://myspace.com/" + name,+ e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + \+ "<SFURL>https://myspace.com/" + name + "</SFURL>",self.__name__, event)self.notifyListeners(e)@@ -99,7 +100,7 @@ class sfp_myspace(SpiderFootPlugin):if eventName == "SOCIAL_MEDIA":try:network = eventData.split(": ")[0]- url = eventData.split(": ")[1]+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.pyindex ff44c300..e46a7c5d 100644--- a/modules/sfp_neutrinoapi.py+++ b/modules/sfp_neutrinoapi.py@@ -107,6 +107,9 @@ class sfp_neutrinoapi(SpiderFootPlugin):if res['code'] == "400":if data.get('api-error-msg'):self.sf.error("Error: " + data.get('api-error-msg'), False)+ if "EXCEED" in data.get('api-error-msg'):+ self.errorState = True+ return Noneelse:self.sf.error("Error: HTTP 400", False)return Nonediff --git a/modules/sfp_openstreetmap.py b/modules/sfp_openstreetmap.pyindex 984b1248..f4518d10 100644--- a/modules/sfp_openstreetmap.py+++ b/modules/sfp_openstreetmap.py@@ -92,11 +92,13 @@ class sfp_openstreetmap(SpiderFootPlugin):self.sf.debug("Skipping PO BOX address")return None+ rx1 = re.compile(r'^(c/o|care of|attn:|attention:)\s+[0-9a-z\s\.]', flags=re.IGNORECASE)# Remove address prefixes for delivery instructions- address = re.sub(r'^(c/o|care of|attn:|attention:)\s+[0-9a-z\s\.],', r'', address, flags=re.IGNORECASE)+ address = re.sub(rx1, r'', address)+ rx2 = re.compile(r'^(Level|Floor|Suite|Room)\s+[0-9a-z]+,', flags=re.IGNORECASE)# Remove address prefixes known to return no results (floor, level, suite, etc).- address = re.sub(r'^(Level|Floor|Suite|Room)\s+[0-9a-z]+,', r'', address, flags=re.IGNORECASE)+ address = re.sub(rx2, r'', address)# Search for addressdata = self.query(eventData)diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.pyindex 27edbefa..0c598d27 100644--- a/modules/sfp_pgp.py+++ b/modules/sfp_pgp.py@@ -74,7 +74,7 @@ class sfp_pgp(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)# Get e-mail addresses on this domain- if eventName == "DOMAIN_NAME":+ if eventName in ["DOMAIN_NAME", "INTERNET_NAME"]:res = self.sf.fetchUrl(self.opts['keyserver_search1'] + eventData,timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])diff --git a/modules/sfp_pulsedive.py b/modules/sfp_pulsedive.pyindex ece64ac6..d7b23f82 100644--- a/modules/sfp_pulsedive.py+++ b/modules/sfp_pulsedive.py@@ -93,7 +93,6 @@ class sfp_pulsedive(SpiderFootPlugin):try:info = json.loads(res['content'])except Exception as e:- print(str(res['content']))self.sf.error("Error processing JSON response from Pulsedive.", False)return Nonediff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.pyindex c6ce3a16..b31f5e2c 100644--- a/modules/sfp_slideshare.py+++ b/modules/sfp_slideshare.py@@ -63,7 +63,7 @@ class sfp_slideshare(SpiderFootPlugin):# Retrieve profiletry:network = eventData.split(": ")[0]- url = eventData.split(": ")[1]+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.pyindex 9c63acad..2554f920 100644--- a/modules/sfp_twitter.py+++ b/modules/sfp_twitter.py@@ -59,7 +59,7 @@ class sfp_twitter(SpiderFootPlugin):# Retrieve profiletry:network = eventData.split(": ")[0]- url = eventData.split(": ")[1]+ url = eventData.split(": ")[1].replace("<SFURL>", "").replace("</SFURL>", "")except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)