Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Introduction of the generic e-mail address type, to avoid recursion on generic email addresses.

smicalleffdb60f4e9826
diff --git a/modules/sfp_builtwith.py b/modules/sfp_builtwith.py
index 7fbab591..26954f60 100644
--- a/modules/sfp_builtwith.py
+++ b/modules/sfp_builtwith.py
@@ -54,7 +54,7 @@ class sfp_builtwith(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return [ "INTERNET_NAME", "EMAILADDR", "RAW_RIR_DATA",
+ return [ "INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
"WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME" ]
def query(self, t):
@@ -114,14 +114,23 @@ class sfp_builtwith(SpiderFootPlugin):
self.notifyListeners(e)
if nb.get('Email', None):
if self.sf.validEmail(nb['Email']):
- e = SpiderFootEvent("EMAILADDR", nb['Email'],
+ if nb['Email'].split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+ e = SpiderFootEvent(evttype, nb['Email'],
self.__name__, event)
self.notifyListeners(e)
if data['Meta'].get("Emails", []):
for email in data['Meta']['Emails']:
if self.sf.validEmail(email):
- e = SpiderFootEvent("EMAILADDR", email,
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ e = SpiderFootEvent(evttype, email,
self.__name__, event)
self.notifyListeners(e)
diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.py
index caef69ad..660bb6f2 100644
--- a/modules/sfp_clearbit.py
+++ b/modules/sfp_clearbit.py
@@ -50,7 +50,7 @@ class sfp_clearbit(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
return [ "RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",
- "AFFILIATE_INTERNET_NAME", "EMAILADDR" ]
+ "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC" ]
def query(self, t):
ret = None
@@ -159,7 +159,11 @@ class sfp_clearbit(SpiderFootPlugin):
self.notifyListeners(evt)
if 'emailAddresses' in data['company']['site']:
for e in data['company']['site']['emailAddresses']:
- evt = SpiderFootEvent("EMAILADDR", e, self.__name__, event)
+ if e.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+ evt = SpiderFootEvent(evttype, e, self.__name__, event)
self.notifyListeners(evt)
# Get the location of the person, also indicating
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index d7fed1fc..5a0ba6b9 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -45,7 +45,7 @@ class sfp_email(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["EMAILADDR", "AFFILIATE_EMAILADDR"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "AFFILIATE_EMAILADDR"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -73,6 +73,9 @@ class sfp_email(SpiderFootPlugin):
if eventName.startswith("AFFILIATE_"):
evttype = "AFFILIATE_EMAILADDR"
+ else:
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
self.sf.info("Found e-mail address: " + email)
mail = email.strip('.')
diff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.py
index c22321cd..4134728a 100644
--- a/modules/sfp_emailcrawlr.py
+++ b/modules/sfp_emailcrawlr.py
@@ -49,7 +49,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ["RAW_RIR_DATA", "EMAILADDR", "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]
+ return ["RAW_RIR_DATA", "EMAILADDR", "EMAILADDR_GENERIC",
+ "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]
# Query domain
# https://emailcrawlr.com/docs
@@ -153,7 +154,12 @@ class sfp_emailcrawlr(SpiderFootPlugin):
if email:
mail_domain = email.lower().split('@')[1]
if self.getTarget().matches(mail_domain, includeChildren=True):
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
name = res.get('name')
diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.py
index 9f0deb57..7b73b280 100644
--- a/modules/sfp_emailformat.py
+++ b/modules/sfp_emailformat.py
@@ -45,7 +45,7 @@ class sfp_emailformat(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["EMAILADDR"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -68,8 +68,6 @@ class sfp_emailformat(SpiderFootPlugin):
emails = self.sf.parseEmails(res['content'])
for email in emails:
- evttype = "EMAILADDR"
-
# Skip unrelated emails
mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
@@ -82,6 +80,11 @@ class sfp_emailformat(SpiderFootPlugin):
continue
self.sf.info("Found e-mail address: " + email)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
diff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.py
index 3fa1c5f9..97790379 100644
--- a/modules/sfp_flickr.py
+++ b/modules/sfp_flickr.py
@@ -52,7 +52,8 @@ class sfp_flickr(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ["EMAILADDR", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "INTERNET_NAME",
+ "DOMAIN_NAME", "LINKED_URL_INTERNAL"]
# Retrieve API key
def retrieveApiKey(self):
@@ -184,7 +185,12 @@ class sfp_flickr(SpiderFootPlugin):
continue
self.sf.info("Found e-mail address: " + email)
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
self.results[email] = True
diff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.py
index 1d14f4d1..82907075 100644
--- a/modules/sfp_fullcontact.py
+++ b/modules/sfp_fullcontact.py
@@ -53,8 +53,8 @@ class sfp_fullcontact(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return [ "EMAILADDR", "RAW_RIR_DATA", "PHONE_NUMBER",
- "GEOINFO", "PHYSICAL_ADDRESS" ]
+ return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",
+ "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS" ]
def query(self, url, data, failcount=0):
header = "Bearer " + self.opts['api_key']
@@ -155,7 +155,12 @@ class sfp_fullcontact(SpiderFootPlugin):
data = data['details']
if data.get("emails"):
for r in data['emails']:
- e = SpiderFootEvent("EMAILADDR", r['value'], self.__name__, event)
+ if r['value'].split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ e = SpiderFootEvent(evttype, r['value'], self.__name__, event)
self.notifyListeners(e)
if data.get("phones"):
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index f61c1650..7e43a561 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -46,7 +46,7 @@ class sfp_gravatar(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',
- 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO',
+ 'EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO',
'ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA']
# Query Gravatar API for the specified email address
@@ -137,8 +137,16 @@ class sfp_gravatar(SpiderFootPlugin):
if data.get('emails') is not None:
for email in data.get('emails'):
- if self.sf.validEmail(email.get('value')) and email.get('value') != eventData:
- evt = SpiderFootEvent("EMAILADDR", email.get('value'), self.__name__, event)
+ em = email.get('value')
+ if not em:
+ continue
+ if self.sf.validEmail(em) and em != eventData:
+ if em.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ evt = SpiderFootEvent(evttype, em, self.__name__, event)
self.notifyListeners(evt)
if data.get('ims') is not None:
diff --git a/modules/sfp_grep_app.py b/modules/sfp_grep_app.py
index aaba31a5..97ee3dda 100644
--- a/modules/sfp_grep_app.py
+++ b/modules/sfp_grep_app.py
@@ -48,7 +48,8 @@ class sfp_grep_app(SpiderFootPlugin):
return ["DOMAIN_NAME"]
def producedEvents(self):
- return ["EMAILADDR", "DOMAIN_NAME", "INTERNET_NAME", "RAW_RIR_DATA",
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "DOMAIN_NAME",
+ "INTERNET_NAME", "RAW_RIR_DATA",
"INTERNET_NAME_UNRESOLVED", "LINKED_URL_INTERNAL"]
def query(self, qry, page):
@@ -188,7 +189,12 @@ class sfp_grep_app(SpiderFootPlugin):
continue
self.sf.info("Found e-mail address: " + email)
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
self.results[email] = True
diff --git a/modules/sfp_hunter.py b/modules/sfp_hunter.py
index 10ff3412..24350019 100644
--- a/modules/sfp_hunter.py
+++ b/modules/sfp_hunter.py
@@ -49,7 +49,7 @@ class sfp_hunter(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return [ "EMAILADDR", "RAW_RIR_DATA" ]
+ return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA" ]
def query(self, t, offset=0, limit=10):
ret = None
@@ -115,7 +115,15 @@ class sfp_hunter(SpiderFootPlugin):
while rescount <= maxgoal:
for email in data['data'].get('emails', list()):
# Notify other modules of what you've found
- e = SpiderFootEvent("EMAILADDR", email['value'], self.__name__, event)
+ em = email.get('value')
+ if not em:
+ continue
+ if em.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ e = SpiderFootEvent(evttype, em, self.__name__, event)
self.notifyListeners(e)
if 'first_name' in email and 'last_name' in email:
diff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.py
index 2872ce3d..f7e0b364 100644
--- a/modules/sfp_jsonwhoiscom.py
+++ b/modules/sfp_jsonwhoiscom.py
@@ -49,8 +49,8 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
return ["RAW_RIR_DATA", "DOMAIN_REGISTRAR", "DOMAIN_WHOIS", "PROVIDER_DNS",
- "EMAILADDR", "PHONE_NUMBER", "PHYSICAL_ADDRESS", "HUMAN_NAME",
- "AFFILIATE_DOMAIN_UNREGISTERED"]
+ "EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "PHYSICAL_ADDRESS",
+ "HUMAN_NAME", "AFFILIATE_DOMAIN_UNREGISTERED"]
# Query domain
# https://jsonwhois.com/docs
@@ -201,7 +201,11 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
for email in set(emails):
mail_domain = email.lower().split('@')[1]
if self.getTarget().matches(mail_domain, includeChildren=True):
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADR"
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
else:
evt = SpiderFootEvent("AFFILIATE_EMAILADDR", email, self.__name__, event)
diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.py
index 0e0504e5..445a5068 100644
--- a/modules/sfp_pgp.py
+++ b/modules/sfp_pgp.py
@@ -53,7 +53,7 @@ class sfp_pgp(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["EMAILADDR", "AFFILIATE_EMAILADDR", "PGP_KEY"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC", "AFFILIATE_EMAILADDR", "PGP_KEY"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -82,7 +82,10 @@ class sfp_pgp(SpiderFootPlugin):
if res['content'] is not None and res['code'] != "503":
emails = self.sf.parseEmails(res['content'])
for email in emails:
- evttype = "EMAILADDR"
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.py
index 680a6429..c30e2176 100644
--- a/modules/sfp_skymem.py
+++ b/modules/sfp_skymem.py
@@ -45,7 +45,7 @@ class sfp_skymem(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["EMAILADDR"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -78,7 +78,11 @@ class sfp_skymem(SpiderFootPlugin):
self.sf.info("Found e-mail address: " + email)
if email not in self.results:
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
self.results[email] = True
@@ -106,7 +110,11 @@ class sfp_skymem(SpiderFootPlugin):
self.sf.info("Found e-mail address: " + email)
if email not in self.results:
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
self.results[email] = True
diff --git a/modules/sfp_snov.py b/modules/sfp_snov.py
index e9bf3d6a..0651fa24 100644
--- a/modules/sfp_snov.py
+++ b/modules/sfp_snov.py
@@ -50,7 +50,7 @@ class sfp_snov(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ["EMAILADDR"]
+ return ["EMAILADDR", "EMAILADDR_GENERIC"]
# Get Authentication token from Snov.IO API
def queryAccessToken(self):
@@ -175,7 +175,7 @@ class sfp_snov(SpiderFootPlugin):
if records:
for record in records:
if record:
- email = record.get('email')
+ email = str(record.get('email'))
if email:
if email in self.results:
continue
@@ -183,7 +183,12 @@ class sfp_snov(SpiderFootPlugin):
continue
self.results[email] = True
- evt = SpiderFootEvent("EMAILADDR", str(email), self.__name__, event)
+ if email.split("@")[0] in self.opts['_genericusers'].split(","):
+ evttype = "EMAILADDR_GENERIC"
+ else:
+ evttype = "EMAILADDR"
+
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
# Determine whether the next offset can have data or not
diff --git a/sf.py b/sf.py
index cd38ca48..c0ac5ec5 100755
--- a/sf.py
+++ b/sf.py
@@ -45,6 +45,7 @@ sfConfig = {
'_fetchtimeout': 5, # number of seconds before giving up on a fetch
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,noc,no-reply,noreply,null,phish,phishing,postmaster,privacy,registrar,root,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
'__version__': '3.0',
'__database': 'spiderfoot.db',
'__webaddr': '127.0.0.1',
@@ -64,6 +65,7 @@ sfConfig = {
sfOptdescs = {
'_debug': "Enable debugging?",
'_internettlds': "List of Internet TLDs.",
+ '_genericusers': "List of usernames that if found as usernames or as part of e-mail addresses, should be treated differently to non-generics.",
'_internettlds_cache': "Hours to cache the Internet TLD list. This can safely be quite a long time given that the list doesn't change too often.",
'_useragent': "User-Agent string to use for HTTP requests. Prefix with an '@' to randomly select the User Agent from a file containing user agent strings for each request, e.g. @C:\\useragents.txt or @/home/bob/useragents.txt. Or supply a URL to load the list from there.",
'_dnsserver': "Override the default resolver with another DNS server. For example, 8.8.8.8 is Google's open DNS server.",
diff --git a/sfdb.py b/sfdb.py
index 858ec076..53263ae3 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -149,6 +149,7 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_GENERIC', 'Email Address - Generic', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",