Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Cleaned up handling of social media accounts.

smicallef078ff2e8d379
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 98eaae7b..de029971 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -84,8 +84,7 @@ class sfp_accounts(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED",
- "ACCOUNT_EXTERNAL_USER_SHARED"]
+ return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED"]
def checkSite(self, name, site):
if 'check_uri' not in site:
@@ -251,7 +250,7 @@ class sfp_accounts(SpiderFootPlugin):
self.notifyListeners(evt)
users.append(kw)
- if eventName == "EMAILADDR":
+ if eventName in ["EMAILADDR", "USERNAME"]:
name = eventData.split("@")[0].lower()
adduser = True
if self.opts['generic'] is list() and name in self.opts['generic']:
@@ -270,7 +269,7 @@ class sfp_accounts(SpiderFootPlugin):
users.append(name)
for user in users:
- if user not in self.reportedUsers:
+ if user not in self.reportedUsers and user != eventData:
evt = SpiderFootEvent("USERNAME", user, self.__name__, event)
self.notifyListeners(evt)
self.reportedUsers.append(user)
diff --git a/modules/sfp_base64.py b/modules/sfp_base64.py
index 1fb524a5..af991058 100644
--- a/modules/sfp_base64.py
+++ b/modules/sfp_base64.py
@@ -56,6 +56,9 @@ class sfp_base64(SpiderFootPlugin):
pat = re.compile("([A-Za-z0-9+\/]+\=\=|[A-Za-z0-9+\/]+\=)")
m = re.findall(pat, eventData)
for match in m:
+ if self.checkForStop():
+ return None
+
minlen = int(self.opts['minlength'])
if len(match) >= minlen:
caps = sum(1 for c in match if c.isupper())
diff --git a/modules/sfp_github.py b/modules/sfp_github.py
index 596761e6..7d1029e1 100644
--- a/modules/sfp_github.py
+++ b/modules/sfp_github.py
@@ -83,14 +83,21 @@ class sfp_github(SpiderFootPlugin):
if eventName == "SOCIAL_MEDIA":
try:
network = eventData.split(": ")[0]
- name = eventData.split(": ")[1]
+ url = eventData.split(": ")[1]
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
return None
if not network == "Github":
- self.sf.debug("Skipping social network profile, " + name + ", as not a Github profile")
+ self.sf.debug("Skipping social network profile, " + url + ", as not a Github profile")
+ return None
+
+ try:
+ bits = url.split("/")
+ name = bits[len(bits)-1]
+ except BaseException as e:
+ self.sf.debug("Couldn't get a username out of " + url)
return None
res = self.sf.fetchUrl("https://api.github.com/users/" + name, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 4a20d371..0474af4c 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -29,11 +29,13 @@ class sfp_gravatar(SpiderFootPlugin):
}
results = dict()
+ reportedUsers = dict()
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.__dataSource__ = 'Gravatar'
self.results = dict()
+ self.reportedUsers = dict()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -44,7 +46,9 @@ class sfp_gravatar(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME', 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO']
+ return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',
+ 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO',
+ 'ACCOUNT_EXTERNAL_OWNED']
# Query Gravatar API for the specified email address
# https://secure.gravatar.com/site/implement/
@@ -100,8 +104,10 @@ class sfp_gravatar(SpiderFootPlugin):
self.notifyListeners(evt)
if data.get('preferredUsername') is not None:
- evt = SpiderFootEvent("USERNAME", data.get('preferredUsername'), self.__name__, event)
+ un = data.get('preferredUsername')
+ evt = SpiderFootEvent("USERNAME", un, self.__name__, event)
self.notifyListeners(evt)
+ self.reportedUsers[un] = True
if data.get('name') is not None and data.get('name').get('formatted') is not None:
evt = SpiderFootEvent("HUMAN_NAME", data.get('name').get('formatted'), self.__name__, event)
@@ -124,20 +130,26 @@ class sfp_gravatar(SpiderFootPlugin):
if data.get('emails') is not None:
for email in data.get('emails'):
- if email.get('value') is not None:
+ if email.get('value') is not None and email.get('value') != eventData:
evt = SpiderFootEvent("EMAILADDR", email.get('value'), self.__name__, event)
self.notifyListeners(evt)
if data.get('ims') is not None:
for im in data.get('ims'):
if im.get('value') is not None:
- evt = SpiderFootEvent("USERNAME", im.get('value'), self.__name__, event)
+ t = im.get('type').capitalize() + " (Instant Messenger)\n" + im.get('value')
+ evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", t, self.__name__, event)
self.notifyListeners(evt)
if data.get('accounts') is not None:
for account in data.get('accounts'):
- if account.get('username') is not None:
- evt = SpiderFootEvent("USERNAME", account.get('username'), self.__name__, event)
+ # For some reason facebook.com gets reported as a username
+ un = account.get('username')
+ if un == "facebook.com":
+ continue
+ if un is not None and un not in self.reportedUsers:
+ evt = SpiderFootEvent("USERNAME", un, self.__name__, event)
self.notifyListeners(evt)
+ self.reportedUsers[un] = True
# End of sfp_gravatar class
diff --git a/modules/sfp_myspace.py b/modules/sfp_myspace.py
index 384355f8..460081f8 100644
--- a/modules/sfp_myspace.py
+++ b/modules/sfp_myspace.py
@@ -87,25 +87,27 @@ class sfp_myspace(SpiderFootPlugin):
return None
name = matches[0]
-
- e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + name, self.__name__, event)
+ e = SpiderFootEvent("SOCIAL_MEDIA", "MySpace: " + "https://myspace.com/" + name,
+ self.__name__, event)
self.notifyListeners(e)
# Retrieve location from MySpace profile
if eventName == "SOCIAL_MEDIA":
try:
network = eventData.split(": ")[0]
- name = eventData.split(": ")[1]
+ url = eventData.split(": ")[1]
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
return None
if network != "MySpace":
- self.sf.debug("Skipping social network profile, " + name + ", as not a MySpace profile")
+ self.sf.debug("Skipping social network profile, " + url + \
+ ", as not a MySpace profile")
return None
- res = self.sf.fetchUrl("https://myspace.com/" + name, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
if res['content'] is None:
return None
diff --git a/modules/sfp_slideshare.py b/modules/sfp_slideshare.py
index 162493c3..e107fc30 100644
--- a/modules/sfp_slideshare.py
+++ b/modules/sfp_slideshare.py
@@ -59,17 +59,18 @@ class sfp_slideshare(SpiderFootPlugin):
# Retrieve profile
try:
network = eventData.split(": ")[0]
- name = eventData.split(": ")[1]
+ url = eventData.split(": ")[1]
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
return None
if not network == "SlideShare":
- self.sf.debug("Skipping social network profile, " + name + ", as not a SlideShare profile")
+ self.sf.debug("Skipping social network profile, " + url + \
+ ", as not a SlideShare profile")
return None
- res = self.sf.fetchUrl("https://slideshare.net/" + name, timeout=self.opts['_fetchtimeout'],
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
if res['content'] is None:
@@ -79,10 +80,11 @@ class sfp_slideshare(SpiderFootPlugin):
human_name = self.extractMeta('slideshare:name', res['content'])
if not human_name:
- self.sf.debug(name + " is not a valid SlideShare profile")
+ self.sf.debug(url + " is not a valid SlideShare profile")
return None
- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0], self.__name__, event)
+ e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + \
+ human_name[0], self.__name__, event)
self.notifyListeners(e)
# Retrieve location (country)
diff --git a/modules/sfp_social.py b/modules/sfp_social.py
index 5852099b..559a712d 100644
--- a/modules/sfp_social.py
+++ b/modules/sfp_social.py
@@ -24,7 +24,8 @@ regexps = dict({
"Twitter": list(['.*twitter.com/([a-zA-Z0-9_]{1,15}$)',
'.*twitter.com/#!/([a-zA-Z0-9_]{1,15}$)'
]),
- "SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)'])
+ "SlideShare": list(['.*slideshare.net/([a-zA-Z0-9_]+$)']),
+ "Instagram": list(['.*instagram.com/([a-zA-Z0-9_]+$)'])
})
@@ -61,7 +62,7 @@ class sfp_social(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["SOCIAL_MEDIA"]
+ return ["SOCIAL_MEDIA", "USERNAME"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -82,9 +83,15 @@ class sfp_social(SpiderFootPlugin):
if bits is not None:
self.sf.info("Matched " + regexpGrp + " in " + eventData)
evt = SpiderFootEvent("SOCIAL_MEDIA", regexpGrp + ": " +
- bits.group(1), self.__name__, event)
+ eventData, self.__name__, event)
self.notifyListeners(evt)
+ # Except for Google+, the link includes potential usernames
+ if regexpGrp != "Google+":
+ un = bits.group(1)
+ evt = SpiderFootEvent("USERNAME", bits.group(1), self.__name__, event)
+ self.notifyListeners(evt)
+
return None
# If you intend for this module to act on its own (e.g. not solely rely
diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.py
index 0c49768b..75f3ffec 100644
--- a/modules/sfp_socialprofiles.py
+++ b/modules/sfp_socialprofiles.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name: sfp_socialprofiles
-# Purpose: Obtains social media profiles of any identified human names.
+# Purpose: Tries to discover social media profiles of any identified human names.
#
# Author: Steve Micallef <[email protected]>
#
@@ -32,7 +32,7 @@ sites = {
class sfp_socialprofiles(SpiderFootPlugin):
- """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Identify the social media profiles for human names identified."""
+ """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Tries to discover the social media profiles for human names identified."""
# Default options
@@ -120,10 +120,6 @@ class sfp_socialprofiles(SpiderFootPlugin):
if self.checkForStop():
return None
- pauseSecs = random.randint(4, 15)
- self.sf.debug("Pausing for " + str(pauseSecs))
- time.sleep(pauseSecs)
-
for key in results:
instances = list()
diff --git a/modules/sfp_twitter.py b/modules/sfp_twitter.py
index cea5f174..ff1a6521 100644
--- a/modules/sfp_twitter.py
+++ b/modules/sfp_twitter.py
@@ -55,31 +55,33 @@ class sfp_twitter(SpiderFootPlugin):
# Retrieve profile
try:
network = eventData.split(": ")[0]
- name = eventData.split(": ")[1]
+ url = eventData.split(": ")[1]
except BaseException as e:
self.sf.error("Unable to parse SOCIAL_MEDIA: " +
eventData + " (" + str(e) + ")", False)
return None
if not network == "Twitter":
- self.sf.debug("Skipping social network profile, " + name + ", as not a Twitter profile")
+ self.sf.debug("Skipping social network profile, " + url + ", as not a Twitter profile")
return None
- res = self.sf.fetchUrl("https://mobile.twitter.com/" + name, timeout=self.opts['_fetchtimeout'],
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
useragent="SpiderFoot")
if res['content'] is None:
return None
if not res['code'] == "200":
- self.sf.debug(name + " is not a valid Twitter profile")
+ self.sf.debug(url + " is not a valid Twitter profile")
return None
# Retrieve name
- human_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>', res['content'], re.MULTILINE)
+ human_name = re.findall(r'<div class="fullname">([^<]+)\s*</div>',
+ res['content'], re.MULTILINE)
if human_name:
- e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0], self.__name__, event)
+ e = SpiderFootEvent("RAW_RIR_DATA", "Possible full name: " + human_name[0],
+ self.__name__, event)
self.notifyListeners(e)
# Retrieve location
diff --git a/sfdb.py b/sfdb.py
index f9e863bd..f3c4a717 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -93,7 +93,6 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",