Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_accounts: code cleanup

Brendan Colesef51b358ea29
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 3f46e1d7..24825e33 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -76,29 +76,25 @@ class sfp_accounts(SpiderFootPlugin):
if content is None:
url = "https://raw.githubusercontent.com/WebBreacher/WhatsMyName/master/web_accounts_list.json"
data = self.sf.fetchUrl(url, useragent="SpiderFoot")
+
if data['content'] is None:
- self.sf.error("Unable to fetch " + url, False)
+ self.sf.error(f"Unable to fetch {url}")
self.errorState = True
return None
- else:
- self.sf.cachePut("sfaccounts", data['content'])
- content = data['content']
+
+ content = data['content']
+ self.sf.cachePut("sfaccounts", content)
try:
self.sites = [site for site in json.loads(content)['sites'] if site['valid']]
- except BaseException as e:
- self.sf.error(f"Unable to parse social media accounts list: {e}", False)
+ except Exception as e:
+ self.sf.error(f"Unable to parse social media accounts list: {e}")
self.errorState = True
return None
- # What events is this module interested in for input
- # * = be notified about all events.
def watchedEvents(self):
return ["EMAILADDR", "DOMAIN_NAME", "HUMAN_NAME", "USERNAME"]
- # What events this module produces
- # This is to support the end user in selecting modules based on events
- # produced.
def producedEvents(self):
return ["USERNAME", "ACCOUNT_EXTERNAL_OWNED"]
@@ -109,40 +105,47 @@ class sfp_accounts(SpiderFootPlugin):
url = site['check_uri'].format(account=name)
retname = f"{site['name']} (Category: {site['category']})\n<SFURL>{url}</SFURL>"
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'], noLog=True, verify=False)
+ res = self.sf.fetchUrl(
+ url,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ noLog=True,
+ verify=False
+ )
if not res['content']:
with self.lock:
self.siteResults[retname] = False
return
- if res['code']:
- if res['code'].startswith("4") or res['code'].startswith("5"):
+ if res['code'] != site.get('account_existence_code'):
+ with self.lock:
+ self.siteResults[retname] = False
+ return
+
+ if site.get('account_existence_string') not in res['content']:
+ with self.lock:
+ self.siteResults[retname] = False
+ return
+
+ if self.opts['musthavename']:
+ if name.lower() not in res['content'].lower():
+ self.sf.debug(f"Skipping {site['name']} as username not mentioned.")
with self.lock:
self.siteResults[retname] = False
return
- # If we see the existence code and string, then consider the
- # site to be found. Otherwise, assume it's not found. Note that
- # the account_missing_code might be the same as the account_existence_code
- # (e.g. 200).
- found = site.get('account_existence_code') == res['code'] \
- and site.get('account_existence_string') in res['content']
-
- if found and self.opts['musthavename']:
- if name not in res['content']:
- self.sf.debug("Skipping " + site['name'] + " as username not mentioned.")
- found = False
-
# Some sites can't handle periods so treat bob.abc and bob as the same
- if found and "." in name:
+ # TODO: fix this once WhatsMyName has support for usernames with '.'
+ if "." in name:
firstname = name.split(".")[0]
if firstname + "<" in res['content'] or firstname + '"' in res['content']:
- found = False
+ with self.lock:
+ self.siteResults[retname] = False
+ return
with self.lock:
- self.siteResults[retname] = found
+ self.siteResults[retname] = True
def checkSites(self, username, sites=None):
def processSiteQueue(username, queue):
@@ -152,7 +155,7 @@ class sfp_accounts(SpiderFootPlugin):
try:
self.checkSite(username, site)
except Exception as e:
- self.sf.debug(f'thread {threading.current_thread().name} exception {e}')
+ self.sf.debug(f'Thread {threading.current_thread().name} exception: {e}')
except QueueEmpty:
return
@@ -188,7 +191,6 @@ class sfp_accounts(SpiderFootPlugin):
return [site for site, found in self.siteResults.items() if found]
- # Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
@@ -202,13 +204,14 @@ class sfp_accounts(SpiderFootPlugin):
# Skip events coming from me unless they are USERNAME events
if eventName != "USERNAME" and srcModuleName == "sfp_accounts":
+ self.sf.debug(f"Ignoring {eventName}, from self.")
return None
- if eventData not in list(self.results.keys()):
- self.results[eventData] = True
- else:
+ if eventData in list(self.results.keys()):
return None
+ self.results[eventData] = True
+
# If being called for the first time, let's see how trusted the
# sites are by attempting to fetch a garbage user.
if not self.distrustedChecked:
@@ -230,7 +233,7 @@ class sfp_accounts(SpiderFootPlugin):
delsites = list()
for site in res:
sitename = site.split(" (Category:")[0]
- self.sf.debug("Distrusting " + sitename)
+ self.sf.debug(f"Distrusting {sitename}")
delsites.append(sitename)
self.sites = [d for d in self.sites if d['name'] not in delsites]
else:
@@ -259,17 +262,17 @@ class sfp_accounts(SpiderFootPlugin):
if eventName == "USERNAME":
users.append(eventData)
- for user in users:
+ for user in set(users):
if user in self.opts['_genericusers'].split(","):
- self.sf.debug(user + " is a generic account name, skipping.")
+ self.sf.debug(f"{user} is a generic account name, skipping.")
continue
if self.opts['ignorenamedict'] and user in self.commonNames:
- self.sf.debug(user + " is found in our name dictionary, skipping.")
+ self.sf.debug(f"{user} is found in our name dictionary, skipping.")
continue
if self.opts['ignoreworddict'] and user in self.words:
- self.sf.debug(user + " is found in our word dictionary, skipping.")
+ self.sf.debug(f"{user} is found in our word dictionary, skipping.")
continue
if user not in self.reportedUsers and eventData != user:
@@ -284,8 +287,12 @@ class sfp_accounts(SpiderFootPlugin):
if eventName == "USERNAME":
res = self.checkSites(user)
for site in res:
- evt = SpiderFootEvent("ACCOUNT_EXTERNAL_OWNED", site,
- self.__name__, event)
+ evt = SpiderFootEvent(
+ "ACCOUNT_EXTERNAL_OWNED",
+ site,
+ self.__name__,
+ event
+ )
self.notifyListeners(evt)
# End of sfp_accounts class