SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Introduction of the generic e-mail address type, to avoid recursion on generic email addresses.
smicalleffdb60f4e9826
diff --git a/modules/sfp_builtwith.py b/modules/sfp_builtwith.pyindex 7fbab591..26954f60 100644--- a/modules/sfp_builtwith.py+++ b/modules/sfp_builtwith.py@@ -54,7 +54,7 @@ class sfp_builtwith(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return [ "INTERNET_NAME", "EMAILADDR", "RAW_RIR_DATA",+ return [ "INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA","WEBSERVER_TECHNOLOGY", "PHONE_NUMBER", "DOMAIN_NAME" ]def query(self, t):@@ -114,14 +114,23 @@ class sfp_builtwith(SpiderFootPlugin):self.notifyListeners(e)if nb.get('Email', None):if self.sf.validEmail(nb['Email']):- e = SpiderFootEvent("EMAILADDR", nb['Email'],+ if nb['Email'].split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"+ e = SpiderFootEvent(evttype, nb['Email'],self.__name__, event)self.notifyListeners(e)if data['Meta'].get("Emails", []):for email in data['Meta']['Emails']:if self.sf.validEmail(email):- e = SpiderFootEvent("EMAILADDR", email,+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ e = SpiderFootEvent(evttype, email,self.__name__, event)self.notifyListeners(e)diff --git a/modules/sfp_clearbit.py b/modules/sfp_clearbit.pyindex caef69ad..660bb6f2 100644--- a/modules/sfp_clearbit.py+++ b/modules/sfp_clearbit.py@@ -50,7 +50,7 @@ class sfp_clearbit(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return [ "RAW_RIR_DATA", "PHONE_NUMBER", "PHYSICAL_ADDRESS",- "AFFILIATE_INTERNET_NAME", "EMAILADDR" ]+ "AFFILIATE_INTERNET_NAME", "EMAILADDR", "EMAILADDR_GENERIC" ]def query(self, t):ret = None@@ -159,7 +159,11 @@ class sfp_clearbit(SpiderFootPlugin):self.notifyListeners(evt)if 'emailAddresses' in data['company']['site']:for e in data['company']['site']['emailAddresses']:- evt = SpiderFootEvent("EMAILADDR", e, self.__name__, event)+ if e.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"+ evt = SpiderFootEvent(evttype, e, self.__name__, event)self.notifyListeners(evt)# Get the location of the person, also indicatingdiff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex d7fed1fc..5a0ba6b9 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -45,7 +45,7 @@ class sfp_email(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["EMAILADDR", "AFFILIATE_EMAILADDR"]+ return ["EMAILADDR", "EMAILADDR_GENERIC", "AFFILIATE_EMAILADDR"]# Handle events sent to this moduledef handleEvent(self, event):@@ -73,6 +73,9 @@ class sfp_email(SpiderFootPlugin):if eventName.startswith("AFFILIATE_"):evttype = "AFFILIATE_EMAILADDR"+ else:+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"self.sf.info("Found e-mail address: " + email)mail = email.strip('.')diff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.pyindex c22321cd..4134728a 100644--- a/modules/sfp_emailcrawlr.py+++ b/modules/sfp_emailcrawlr.py@@ -49,7 +49,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ["RAW_RIR_DATA", "EMAILADDR", "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]+ return ["RAW_RIR_DATA", "EMAILADDR", "EMAILADDR_GENERIC",+ "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]# Query domain# https://emailcrawlr.com/docs@@ -153,7 +154,12 @@ class sfp_emailcrawlr(SpiderFootPlugin):if email:mail_domain = email.lower().split('@')[1]if self.getTarget().matches(mail_domain, includeChildren=True):- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)name = res.get('name')diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.pyindex 9f0deb57..7b73b280 100644--- a/modules/sfp_emailformat.py+++ b/modules/sfp_emailformat.py@@ -45,7 +45,7 @@ class sfp_emailformat(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["EMAILADDR"]+ return ["EMAILADDR", "EMAILADDR_GENERIC"]# Handle events sent to this moduledef handleEvent(self, event):@@ -68,8 +68,6 @@ class sfp_emailformat(SpiderFootPlugin):emails = self.sf.parseEmails(res['content'])for email in emails:- evttype = "EMAILADDR"-# Skip unrelated emailsmailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):@@ -82,6 +80,11 @@ class sfp_emailformat(SpiderFootPlugin):continueself.sf.info("Found e-mail address: " + email)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"+evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)diff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.pyindex 3fa1c5f9..97790379 100644--- a/modules/sfp_flickr.py+++ b/modules/sfp_flickr.py@@ -52,7 +52,8 @@ class sfp_flickr(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ["EMAILADDR", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"]+ return ["EMAILADDR", "EMAILADDR_GENERIC", "INTERNET_NAME",+ "DOMAIN_NAME", "LINKED_URL_INTERNAL"]# Retrieve API keydef retrieveApiKey(self):@@ -184,7 +185,12 @@ class sfp_flickr(SpiderFootPlugin):continueself.sf.info("Found e-mail address: " + email)- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)self.results[email] = Truediff --git a/modules/sfp_fullcontact.py b/modules/sfp_fullcontact.pyindex 1d14f4d1..82907075 100644--- a/modules/sfp_fullcontact.py+++ b/modules/sfp_fullcontact.py@@ -53,8 +53,8 @@ class sfp_fullcontact(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return [ "EMAILADDR", "RAW_RIR_DATA", "PHONE_NUMBER",- "GEOINFO", "PHYSICAL_ADDRESS" ]+ return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA",+ "PHONE_NUMBER", "GEOINFO", "PHYSICAL_ADDRESS" ]def query(self, url, data, failcount=0):header = "Bearer " + self.opts['api_key']@@ -155,7 +155,12 @@ class sfp_fullcontact(SpiderFootPlugin):data = data['details']if data.get("emails"):for r in data['emails']:- e = SpiderFootEvent("EMAILADDR", r['value'], self.__name__, event)+ if r['value'].split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ e = SpiderFootEvent(evttype, r['value'], self.__name__, event)self.notifyListeners(e)if data.get("phones"):diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex f61c1650..7e43a561 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -46,7 +46,7 @@ class sfp_gravatar(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',- 'EMAILADDR', 'PHONE_NUMBER', 'GEOINFO',+ 'EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO','ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA']# Query Gravatar API for the specified email address@@ -137,8 +137,16 @@ class sfp_gravatar(SpiderFootPlugin):if data.get('emails') is not None:for email in data.get('emails'):- if self.sf.validEmail(email.get('value')) and email.get('value') != eventData:- evt = SpiderFootEvent("EMAILADDR", email.get('value'), self.__name__, event)+ em = email.get('value')+ if not em:+ continue+ if self.sf.validEmail(em) and em != eventData:+ if em.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ evt = SpiderFootEvent(evttype, em, self.__name__, event)self.notifyListeners(evt)if data.get('ims') is not None:diff --git a/modules/sfp_grep_app.py b/modules/sfp_grep_app.pyindex aaba31a5..97ee3dda 100644--- a/modules/sfp_grep_app.py+++ b/modules/sfp_grep_app.py@@ -48,7 +48,8 @@ class sfp_grep_app(SpiderFootPlugin):return ["DOMAIN_NAME"]def producedEvents(self):- return ["EMAILADDR", "DOMAIN_NAME", "INTERNET_NAME", "RAW_RIR_DATA",+ return ["EMAILADDR", "EMAILADDR_GENERIC", "DOMAIN_NAME",+ "INTERNET_NAME", "RAW_RIR_DATA","INTERNET_NAME_UNRESOLVED", "LINKED_URL_INTERNAL"]def query(self, qry, page):@@ -188,7 +189,12 @@ class sfp_grep_app(SpiderFootPlugin):continueself.sf.info("Found e-mail address: " + email)- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)self.results[email] = Truediff --git a/modules/sfp_hunter.py b/modules/sfp_hunter.pyindex 10ff3412..24350019 100644--- a/modules/sfp_hunter.py+++ b/modules/sfp_hunter.py@@ -49,7 +49,7 @@ class sfp_hunter(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return [ "EMAILADDR", "RAW_RIR_DATA" ]+ return [ "EMAILADDR", "EMAILADDR_GENERIC", "RAW_RIR_DATA" ]def query(self, t, offset=0, limit=10):ret = None@@ -115,7 +115,15 @@ class sfp_hunter(SpiderFootPlugin):while rescount <= maxgoal:for email in data['data'].get('emails', list()):# Notify other modules of what you've found- e = SpiderFootEvent("EMAILADDR", email['value'], self.__name__, event)+ em = email.get('value')+ if not em:+ continue+ if em.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ e = SpiderFootEvent(evttype, em, self.__name__, event)self.notifyListeners(e)if 'first_name' in email and 'last_name' in email:diff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.pyindex 2872ce3d..f7e0b364 100644--- a/modules/sfp_jsonwhoiscom.py+++ b/modules/sfp_jsonwhoiscom.py@@ -49,8 +49,8 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return ["RAW_RIR_DATA", "DOMAIN_REGISTRAR", "DOMAIN_WHOIS", "PROVIDER_DNS",- "EMAILADDR", "PHONE_NUMBER", "PHYSICAL_ADDRESS", "HUMAN_NAME",- "AFFILIATE_DOMAIN_UNREGISTERED"]+ "EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "PHYSICAL_ADDRESS",+ "HUMAN_NAME", "AFFILIATE_DOMAIN_UNREGISTERED"]# Query domain# https://jsonwhois.com/docs@@ -201,7 +201,11 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):for email in set(emails):mail_domain = email.lower().split('@')[1]if self.getTarget().matches(mail_domain, includeChildren=True):- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADR"+ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)else:evt = SpiderFootEvent("AFFILIATE_EMAILADDR", email, self.__name__, event)diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.pyindex 0e0504e5..445a5068 100644--- a/modules/sfp_pgp.py+++ b/modules/sfp_pgp.py@@ -53,7 +53,7 @@ class sfp_pgp(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["EMAILADDR", "AFFILIATE_EMAILADDR", "PGP_KEY"]+ return ["EMAILADDR", "EMAILADDR_GENERIC", "AFFILIATE_EMAILADDR", "PGP_KEY"]# Handle events sent to this moduledef handleEvent(self, event):@@ -82,7 +82,10 @@ class sfp_pgp(SpiderFootPlugin):if res['content'] is not None and res['code'] != "503":emails = self.sf.parseEmails(res['content'])for email in emails:- evttype = "EMAILADDR"+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"mailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.pyindex 680a6429..c30e2176 100644--- a/modules/sfp_skymem.py+++ b/modules/sfp_skymem.py@@ -45,7 +45,7 @@ class sfp_skymem(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["EMAILADDR"]+ return ["EMAILADDR", "EMAILADDR_GENERIC"]# Handle events sent to this moduledef handleEvent(self, event):@@ -78,7 +78,11 @@ class sfp_skymem(SpiderFootPlugin):self.sf.info("Found e-mail address: " + email)if email not in self.results:- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"+ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)self.results[email] = True@@ -106,7 +110,11 @@ class sfp_skymem(SpiderFootPlugin):self.sf.info("Found e-mail address: " + email)if email not in self.results:- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"+ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)self.results[email] = Truediff --git a/modules/sfp_snov.py b/modules/sfp_snov.pyindex e9bf3d6a..0651fa24 100644--- a/modules/sfp_snov.py+++ b/modules/sfp_snov.py@@ -50,7 +50,7 @@ class sfp_snov(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ["EMAILADDR"]+ return ["EMAILADDR", "EMAILADDR_GENERIC"]# Get Authentication token from Snov.IO APIdef queryAccessToken(self):@@ -175,7 +175,7 @@ class sfp_snov(SpiderFootPlugin):if records:for record in records:if record:- email = record.get('email')+ email = str(record.get('email'))if email:if email in self.results:continue@@ -183,7 +183,12 @@ class sfp_snov(SpiderFootPlugin):continueself.results[email] = True- evt = SpiderFootEvent("EMAILADDR", str(email), self.__name__, event)+ if email.split("@")[0] in self.opts['_genericusers'].split(","):+ evttype = "EMAILADDR_GENERIC"+ else:+ evttype = "EMAILADDR"++ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)# Determine whether the next offset can have data or notdiff --git a/sf.py b/sf.pyindex cd38ca48..c0ac5ec5 100755--- a/sf.py+++ b/sf.py@@ -45,6 +45,7 @@ sfConfig = {'_fetchtimeout': 5, # number of seconds before giving up on a fetch'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat','_internettlds_cache': 72,+ '_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,noc,no-reply,noreply,null,phish,phishing,postmaster,privacy,registrar,root,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",'__version__': '3.0','__database': 'spiderfoot.db','__webaddr': '127.0.0.1',@@ -64,6 +65,7 @@ sfConfig = {sfOptdescs = {'_debug': "Enable debugging?",'_internettlds': "List of Internet TLDs.",+ '_genericusers': "List of usernames that if found as usernames or as part of e-mail addresses, should be treated differently to non-generics.",'_internettlds_cache': "Hours to cache the Internet TLD list. This can safely be quite a long time given that the list doesn't change too often.",'_useragent': "User-Agent string to use for HTTP requests. Prefix with an '@' to randomly select the User Agent from a file containing user agent strings for each request, e.g. @C:\\useragents.txt or @/home/bob/useragents.txt. Or supply a URL to load the list from there.",'_dnsserver': "Override the default resolver with another DNS server. For example, 8.8.8.8 is Google's open DNS server.",diff --git a/sfdb.py b/sfdb.pyindex 858ec076..53263ae3 100644--- a/sfdb.py+++ b/sfdb.py@@ -149,6 +149,7 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_GENERIC', 'Email Address - Generic', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",