SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Cleaned up handling of social media accounts.
smicallef078ff2e8d379
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 98eaae7b..de029971 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -84,8 +84,7 @@ class sfp_accounts(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED",- "ACCOUNT_EXTERNAL_USER_SHARED"]+ return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED"]def checkSite(self, name, site):if 'check_uri' not in site:@@ -251,7 +250,7 @@ class sfp_accounts(SpiderFootPlugin):self.notifyListeners(evt)users.append(kw)- if eventName == "EMAILADDR":+ if eventName in ["EMAILADDR", "USERNAME"]:name = eventData.split("@")[0].lower()adduser = Trueif self.opts['generic'] is list() and name in self.opts['generic']:@@ -270,7 +269,7 @@ class sfp_accounts(SpiderFootPlugin):users.append(name)for user in users:- if user not in self.reportedUsers:+ if user not in self.reportedUsers and user != eventData:evt = SpiderFootEvent("USERNAME", user, self.__name__, event)self.notifyListeners(evt)self.reportedUsers.append(user)diff --git a/modules/sfp_base64.py b/modules/sfp_base64.pyindex 1fb524a5..af991058 100644--- a/modules/sfp_base64.py+++ b/modules/sfp_base64.py@@ -56,6 +56,9 @@ class sfp_base64(SpiderFootPlugin):pat = re.compile("([A-Za-z0-9+\/]+\=\=|[A-Za-z0-9+\/]+\=)")m = re.findall(pat, eventData)for match in m:+ if self.checkForStop():+ return None+minlen = int(self.opts['minlength'])if len(match) >= minlen:caps = sum(1 for c in match if c.isupper())diff --git a/modules/sfp_github.py b/modules/sfp_github.pyindex 596761e6..7d1029e1 100644--- a/modules/sfp_github.py+++ b/modules/sfp_github.py@@ -83,14 +83,21 @@ class sfp_github(SpiderFootPlugin):if eventName == "SOCIAL_MEDIA":try:network = eventData.split(": ")[0]- name = eventData.split(": ")[1]+ url = eventData.split(": ")[1]except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)return Noneif not network == "Github":- self.sf.debug("Skipping social network profile, " + name + ", as not a Github profile")+ self.sf.debug("Skipping social network profile, " + url + ", as not a Github profile")+ return None++ try:+ bits = url.split("/")+ name = bits[len(bits)-1]+ except BaseException as e:+ self.sf.debug("Couldn't get a username out of " + url)return Noneres = self.sf.fetchUrl("https://api.github.com/users/" + name, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex 4a20d371..0474af4c 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -29,11 +29,13 @@ class sfp_gravatar(SpiderFootPlugin):}results = dict()+ reportedUsers = dict()def setup(self, sfc, userOpts=dict()):self.sf = sfcself.__dataSource__ = 'Gravatar'self.results = dict()+ self.reportedUsers = dict()for opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -44,7 +46,9 @@ class sfp_gravatar(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME', 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO']+ return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',+ 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO',+ 'ACCOUNT_EXTERNAL_OWNED']# Query Gravatar API for the specified email address# https://secure.gravatar.com/site/implement/@@ -100,8 +104,10 @@ class sfp_gravatar(SpiderFootPlugin):self.notifyListeners(evt)if data.get('preferredUsername') is not None:- evt = SpiderFootEvent("USERNAME", data.get('preferredUsername'), self.__name__, event)+ un = data.get('preferredUsername')+ evt = SpiderFootEvent("USERNAME", un, self.__name__, event)self.notifyListeners(evt)+ self.reportedUsers[un] = Trueif data.get('name') is not None and data.get('name').get('formatted') is not None:evt = SpiderFootEvent("HUMAN_NAME", data.get('name').get('formatted'), self.__name__, event)@@ -124,20 +130,26 @@ class sfp_gravatar(SpiderFootPlugin):if data.get('emails') is not None:for email in data.get('emails'):- if email.get('value') is not None:+ if email.get('value') is not None and email.get('value') != eventData:evt = SpiderFootEvent("EMAILADDR", email.get('value'), self.__name__, event)self.notifyListeners(evt)if data.get('ims') is not None:for im in data.get('ims'):if im.get('value') is not None:- evt = SpiderFootEvent("USERNAME", im.get('value'), self.__name__, event)+ t = im.get('type').capitalize() + " (Instant Messenger)\n" + im.get('value')+ evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", t, self.__name__, event)self.notifyListeners(evt)if data.get('accounts') is not None:for account in data.get('accounts'):- if account.get('username') is not None:- evt = SpiderFootEvent("USERNAME", account.get('username'), self.__name__, event)+ # For some reason facebook.com gets reported as a username+ un = account.get('username')+ if un == "facebook.com":+ continue+ if un is not None and un not in self.reportedUsers:+ evt = SpiderFootEvent("USERNAME", un, self.__name__, event)self.notifyListeners(evt)+ self.reportedUsers[un] = True# End of sfp_gravatar classdiff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.pyindex 384355f8..460081f8 100644--- a/modules/sfp_myspace.py+++ b/modules/sfp_myspace.py@@ -87,25 +87,27 @@ class sfp_myspace(SpiderFootPlugin):return Nonename = matches[0]-- e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + name, self.__name__, event)+ e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + "https://myspace.com/" + name,+ self.__name__, event)self.notifyListeners(e)# Retrieve location from MySpace profileif eventName == "SOCIAL_MEDIA":try:network = eventData.split(": ")[0]- name = eventData.split(": ")[1]+ url = eventData.split(": ")[1]except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)return Noneif network != "MySpace":- self.sf.debug("Skipping social network profile, " + name + ", as not a MySpace profile")+ self.sf.debug("Skipping social network profile, " + url + \+ ", as not a MySpace profile")return None- res = self.sf.fetchUrl("https://myspace.com/" + name, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])if res['content'] is None:return Nonediff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.pyindex 162493c3..e107fc30 100644--- a/modules/sfp_slideshare.py+++ b/modules/sfp_slideshare.py@@ -59,17 +59,18 @@ class sfp_slideshare(SpiderFootPlugin):# Retrieve profiletry:network = eventData.split(": ")[0]- name = eventData.split(": ")[1]+ url = eventData.split(": ")[1]except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)return Noneif not network == "SlideShare":- self.sf.debug("Skipping social network profile, " + name + ", as not a SlideShare profile")+ self.sf.debug("Skipping social network profile, " + url + \+ ", as not a SlideShare profile")return None- res = self.sf.fetchUrl("https://slideshare.net/" + name, timeout=self.opts['_fetchtimeout'],+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent=self.opts['_useragent'])if res['content'] is None:@@ -79,10 +80,11 @@ class sfp_slideshare(SpiderFootPlugin):human_name = self.extractMeta('slideshare:name', res['content'])if not human_name:- self.sf.debug(name + " is not a valid SlideShare profile")+ self.sf.debug(url + " is not a valid SlideShare profile")return None- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0], self.__name__, event)+ e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + \+ human_name[0], self.__name__, event)self.notifyListeners(e)# Retrieve location (country)diff --git a/modules/sfp_social.py b/modules/sfp_social.pyindex 5852099b..559a712d 100644--- a/modules/sfp_social.py+++ b/modules/sfp_social.py@@ -24,7 +24,8 @@ regexps = dict({"Twitter": list(['.*twitter.com/([a-zA-Z0-9_]{1,15}$)','.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)']),- "SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)'])+ "SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)']),+ "Instagram": list(['.*instagram.com/([a-zA-Z0-9_]+$)'])})@@ -61,7 +62,7 @@ class sfp_social(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["SOCIAL_MEDIA"]+ return ["SOCIAL_MEDIA", "USERNAME"]# Handle events sent to this moduledef handleEvent(self, event):@@ -82,9 +83,15 @@ class sfp_social(SpiderFootPlugin):if bits is not None:self.sf.info("Matched " + regexpGrp + " in " + eventData)evt = SpiderFootEvent("SOCIAL_MEDIA", regexpGrp + ": " +- bits.group(1), self.__name__, event)+ eventData, self.__name__, event)self.notifyListeners(evt)+ # Except for Google+, the link includes potential usernames+ if regexpGrp != "Google+":+ un = bits.group(1)+ evt = SpiderFootEvent("USERNAME", bits.group(1), self.__name__, event)+ self.notifyListeners(evt)+return None# If you intend for this module to act on its own (e.g. not solely relydiff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.pyindex 0c49768b..75f3ffec 100644--- a/modules/sfp_socialprofiles.py+++ b/modules/sfp_socialprofiles.py@@ -1,7 +1,7 @@# -*- coding: utf-8 -*-# -------------------------------------------------------------------------------# Name: sfp_socialprofiles-# Purpose: Obtains social media profiles of any identified human names.+# Purpose: Tries to discover social media profiles of any identified human names.## Author: Steve Micallef <[email protected]>#@@ -32,7 +32,7 @@ sites = {class sfp_socialprofiles(SpiderFootPlugin):- """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Identify the social media profiles for human names identified."""+ """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Tries to discover the social media profiles for human names identified."""# Default options@@ -120,10 +120,6 @@ class sfp_socialprofiles(SpiderFootPlugin):if self.checkForStop():return None- pauseSecs = random.randint(4, 15)- self.sf.debug("Pausing for " + str(pauseSecs))- time.sleep(pauseSecs)-for key in results:instances = list()diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.pyindex cea5f174..ff1a6521 100644--- a/modules/sfp_twitter.py+++ b/modules/sfp_twitter.py@@ -55,31 +55,33 @@ class sfp_twitter(SpiderFootPlugin):# Retrieve profiletry:network = eventData.split(": ")[0]- name = eventData.split(": ")[1]+ url = eventData.split(": ")[1]except BaseException as e:self.sf.error("Unable to parse SOCIAL_MEDIA: " +eventData + " (" + str(e) + ")", False)return Noneif not network == "Twitter":- self.sf.debug("Skipping social network profile, " + name + ", as not a Twitter profile")+ self.sf.debug("Skipping social network profile, " + url + ", as not a Twitter profile")return None- res = self.sf.fetchUrl("https://mobile.twitter.com/" + name, timeout=self.opts['_fetchtimeout'],+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],useragent="SpiderFoot")if res['content'] is None:return Noneif not res['code'] == "200":- self.sf.debug(name + " is not a valid Twitter profile")+ self.sf.debug(url + " is not a valid Twitter profile")return None# Retrieve name- human_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>', res['content'], re.MULTILINE)+ human_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>',+ res['content'], re.MULTILINE)if human_name:- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0], self.__name__, event)+ e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0],+ self.__name__, event)self.notifyListeners(e)# Retrieve locationdiff --git a/sfdb.py b/sfdb.pyindex f9e863bd..f3c4a717 100644--- a/sfdb.py+++ b/sfdb.py@@ -93,7 +93,6 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",