SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Add sf.parseEmails function to extract emails from strings
Brendan Coles71f2980f91cd
diff --git a/modules/sfp_email.py b/modules/sfp_email.pyindex 5bf88d12..c8f8a958 100644--- a/modules/sfp_email.py+++ b/modules/sfp_email.py@@ -59,22 +59,13 @@ class sfp_email(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)- pat = re.compile("([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")- matches = re.findall(pat, eventData)+ emails = self.sf.parseEmails(eventData)myres = list()- for match in matches:+ for email in emails:evttype = "EMAILADDR"- if len(match) < 4:- self.sf.debug("Likely invalid address: " + match)- continue-- # Handle messed up encodings- if "%" in match:- self.sf.debug("Skipped address: " + match)- continue# Get the domain and strip potential ending .- mailDom = match.lower().split('@')[1].strip('.')+ mailDom = email.lower().split('@')[1].strip('.')if not self.getTarget().matches(mailDom) and not self.getTarget().matches(match):self.sf.debug("External domain, so possible affiliate e-mail")evttype = "AFFILIATE_EMAILADDR"@@ -82,17 +73,17 @@ class sfp_email(SpiderFootPlugin):if eventName.startswith("AFFILIATE_"):evttype = "AFFILIATE_EMAILADDR"- self.sf.info("Found e-mail address: " + match)- if type(match) == str:- mail = unicode(match.strip('.'), 'utf-8', errors='replace')+ self.sf.info("Found e-mail address: " + email)+ if type(email) == str:+ mail = unicode(email.strip('.'), 'utf-8', errors='replace')else:- mail = match.strip('.')+ mail = email.strip('.')if mail in myres:self.sf.debug("Already found from this source.")continue- else:- myres.append(mail)++ myres.append(mail)evt = SpiderFootEvent(evttype, mail, self.__name__, event)if event.moduleDataSource:diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.pyindex aee67c83..f43e325e 100644--- a/modules/sfp_emailformat.py+++ b/modules/sfp_emailformat.py@@ -67,34 +67,18 @@ class sfp_emailformat(SpiderFootPlugin):if res['content'] is None:return None- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")- matches = re.findall(pat, res['content'])- for match in matches:+ emails = self.sf.parseEmails(res['content'])+ for email in emails:evttype = "EMAILADDR"- self.sf.debug("Found possible email: " + match)-- # Handle false positive matches- if len(match) < 5:- self.sf.debug("Likely invalid address.")- continue-- if "..." in match:- self.sf.debug("Incomplete e-mail address, skipping.")- continue-- # Handle messed up encodings- if "%" in match:- self.sf.debug("Skipped address: " + match)- continue# Skip unrelated emails- mailDom = match.lower().split('@')[1]+ mailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):- self.sf.debug("Skipped address: " + match)+ self.sf.debug("Skipped address: " + email)continue- self.sf.info("Found e-mail address: " + match)- evt = SpiderFootEvent(evttype, match, self.__name__, event)+ self.sf.info("Found e-mail address: " + email)+ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)return Nonediff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.pyindex 52bd519c..23cea913 100644--- a/modules/sfp_pgp.py+++ b/modules/sfp_pgp.py@@ -81,21 +81,16 @@ class sfp_pgp(SpiderFootPlugin):useragent=self.opts['_useragent'])if res['content'] is not None:- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")- matches = re.findall(pat, res['content'])- for match in matches:+ emails = self.sf.parseEmails(res['content'])+ for email in emails:evttype = "EMAILADDR"- self.sf.debug("Found possible email: " + match)- if len(match) < 4:- self.sf.debug("Likely invalid address.")- continue- mailDom = match.lower().split('@')[1]+ mailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):evttype = "AFFILIATE_EMAILADDR"- self.sf.info("Found e-mail address: " + match)- evt = SpiderFootEvent(evttype, match, self.__name__, event)+ self.sf.info("Found e-mail address: " + email)+ evt = SpiderFootEvent(evttype, email, self.__name__, event)self.notifyListeners(evt)if eventName == "EMAILADDR":diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.pyindex d11be02d..05507e50 100644--- a/modules/sfp_skymem.py+++ b/modules/sfp_skymem.py@@ -67,35 +67,20 @@ class sfp_skymem(SpiderFootPlugin):return None# Extract emails from results page- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")- matches = re.findall(pat, res['content'])- if not matches:- return None-- for match in matches:- self.sf.debug("Found possible email: " + match)-- # Handle false positive matches- if len(match) < 5:- self.sf.debug("Likely invalid address.")- continue-- # Handle messed up encodings- if "%" in match:- self.sf.debug("Skipped address: " + match)- continue+ emails = self.sf.parseEmails(res['content'])+ for email in emails:# Skip unrelated emails- mailDom = match.lower().split('@')[1]+ mailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):- self.sf.debug("Skipped address: " + match)+ self.sf.debug("Skipped address: " + email)continue- self.sf.info("Found e-mail address: " + match)- if match not in self.results:- evt = SpiderFootEvent("EMAILADDR", match, self.__name__, event)+ self.sf.info("Found e-mail address: " + email)+ if emnail not in self.results:+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)self.notifyListeners(evt)- self.results[match] = True+ self.results[email] = True# Loop through first 20 pages of resultsdomain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', res['content'])@@ -111,32 +96,19 @@ class sfp_skymem(SpiderFootPlugin):if res['content'] is None:break- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")- matches = re.findall(pat, res['content'])- for match in matches:- self.sf.debug("Found possible email: " + match)-- # Handle false positive matches- if len(match) < 5:- self.sf.debug("Likely invalid address.")- continue-- # Handle messed up encodings- if "%" in match:- self.sf.debug("Skipped address: " + match)- continue-+ emails = self.sf.parseEmails(res['content'])+ for email in emails:# Skip unrelated emails- mailDom = match.lower().split('@')[1]+ mailDom = email.lower().split('@')[1]if not self.getTarget().matches(mailDom):- self.sf.debug("Skipped address: " + match)+ self.sf.debug("Skipped address: " + email)continue- self.sf.info("Found e-mail address: " + match)- if match not in self.results:- evt = SpiderFootEvent("EMAILADDR", match, self.__name__, event)+ self.sf.info("Found e-mail address: " + email)+ if email not in self.results:+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)self.notifyListeners(evt)- self.results[match] = True+ self.results[email] = True# Check if we're on the last page of resultsmax_page = 0diff --git a/sflib.py b/sflib.pyindex d4dec00b..da1e7f87 100644--- a/sflib.py+++ b/sflib.py@@ -889,6 +889,34 @@ class SpiderFoot:return returnArr+ # Find all emails within the supplied content+ # Returns an Array+ def parseEmails(self, data):+ emails = list()+ matches = re.findall(r'([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)', str(data).decode('unicode-escape'))++ for match in matches:+ self.debug("Found possible email: " + match)++ # Handle false positive matches+ if len(match) < 5:+ self.debug("Skipped likely invalid email address.")+ continue++ # Handle messed up encodings+ if "%" in match:+ self.debug("Skipped invalid email address: " + match)+ continue++ # Handle truncated emails+ if "..." in match:+ self.debug("Skipped incomplete e-mail address: " + match)+ continue++ emails.append(match)++ return set(emails)+# Find all URLs within the supplied content. This does not fetch any URLs!# A dictionary will be returned, where each link will have the keys# 'source': The URL where the link was obtained from