Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Add sf.parseEmails function to extract emails from strings

Brendan Coles71f2980f91cd
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index 5bf88d12..c8f8a958 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -59,22 +59,13 @@ class sfp_email(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
- pat = re.compile("([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")
- matches = re.findall(pat, eventData)
+ emails = self.sf.parseEmails(eventData)
myres = list()
- for match in matches:
+ for email in emails:
evttype = "EMAILADDR"
- if len(match) < 4:
- self.sf.debug("Likely invalid address: " + match)
- continue
-
- # Handle messed up encodings
- if "%" in match:
- self.sf.debug("Skipped address: " + match)
- continue
# Get the domain and strip potential ending .
- mailDom = match.lower().split('@')[1].strip('.')
+ mailDom = email.lower().split('@')[1].strip('.')
if not self.getTarget().matches(mailDom) and not self.getTarget().matches(match):
self.sf.debug("External domain, so possible affiliate e-mail")
evttype = "AFFILIATE_EMAILADDR"
@@ -82,17 +73,17 @@ class sfp_email(SpiderFootPlugin):
if eventName.startswith("AFFILIATE_"):
evttype = "AFFILIATE_EMAILADDR"
- self.sf.info("Found e-mail address: " + match)
- if type(match) == str:
- mail = unicode(match.strip('.'), 'utf-8', errors='replace')
+ self.sf.info("Found e-mail address: " + email)
+ if type(email) == str:
+ mail = unicode(email.strip('.'), 'utf-8', errors='replace')
else:
- mail = match.strip('.')
+ mail = email.strip('.')
if mail in myres:
self.sf.debug("Already found from this source.")
continue
- else:
- myres.append(mail)
+
+ myres.append(mail)
evt = SpiderFootEvent(evttype, mail, self.__name__, event)
if event.moduleDataSource:
diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.py
index aee67c83..f43e325e 100644
--- a/modules/sfp_emailformat.py
+++ b/modules/sfp_emailformat.py
@@ -67,34 +67,18 @@ class sfp_emailformat(SpiderFootPlugin):
if res['content'] is None:
return None
- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")
- matches = re.findall(pat, res['content'])
- for match in matches:
+ emails = self.sf.parseEmails(res['content'])
+ for email in emails:
evttype = "EMAILADDR"
- self.sf.debug("Found possible email: " + match)
-
- # Handle false positive matches
- if len(match) < 5:
- self.sf.debug("Likely invalid address.")
- continue
-
- if "..." in match:
- self.sf.debug("Incomplete e-mail address, skipping.")
- continue
-
- # Handle messed up encodings
- if "%" in match:
- self.sf.debug("Skipped address: " + match)
- continue
# Skip unrelated emails
- mailDom = match.lower().split('@')[1]
+ mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
- self.sf.debug("Skipped address: " + match)
+ self.sf.debug("Skipped address: " + email)
continue
- self.sf.info("Found e-mail address: " + match)
- evt = SpiderFootEvent(evttype, match, self.__name__, event)
+ self.sf.info("Found e-mail address: " + email)
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
return None
diff --git a/modules/sfp_pgp.py b/modules/sfp_pgp.py
index 52bd519c..23cea913 100644
--- a/modules/sfp_pgp.py
+++ b/modules/sfp_pgp.py
@@ -81,21 +81,16 @@ class sfp_pgp(SpiderFootPlugin):
useragent=self.opts['_useragent'])
if res['content'] is not None:
- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")
- matches = re.findall(pat, res['content'])
- for match in matches:
+ emails = self.sf.parseEmails(res['content'])
+ for email in emails:
evttype = "EMAILADDR"
- self.sf.debug("Found possible email: " + match)
- if len(match) < 4:
- self.sf.debug("Likely invalid address.")
- continue
- mailDom = match.lower().split('@')[1]
+ mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
evttype = "AFFILIATE_EMAILADDR"
- self.sf.info("Found e-mail address: " + match)
- evt = SpiderFootEvent(evttype, match, self.__name__, event)
+ self.sf.info("Found e-mail address: " + email)
+ evt = SpiderFootEvent(evttype, email, self.__name__, event)
self.notifyListeners(evt)
if eventName == "EMAILADDR":
diff --git a/modules/sfp_skymem.py b/modules/sfp_skymem.py
index d11be02d..05507e50 100644
--- a/modules/sfp_skymem.py
+++ b/modules/sfp_skymem.py
@@ -67,35 +67,20 @@ class sfp_skymem(SpiderFootPlugin):
return None
# Extract emails from results page
- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")
- matches = re.findall(pat, res['content'])
- if not matches:
- return None
-
- for match in matches:
- self.sf.debug("Found possible email: " + match)
-
- # Handle false positive matches
- if len(match) < 5:
- self.sf.debug("Likely invalid address.")
- continue
-
- # Handle messed up encodings
- if "%" in match:
- self.sf.debug("Skipped address: " + match)
- continue
+ emails = self.sf.parseEmails(res['content'])
+ for email in emails:
# Skip unrelated emails
- mailDom = match.lower().split('@')[1]
+ mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
- self.sf.debug("Skipped address: " + match)
+ self.sf.debug("Skipped address: " + email)
continue
- self.sf.info("Found e-mail address: " + match)
- if match not in self.results:
- evt = SpiderFootEvent("EMAILADDR", match, self.__name__, event)
+ self.sf.info("Found e-mail address: " + email)
+ if emnail not in self.results:
+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
self.notifyListeners(evt)
- self.results[match] = True
+ self.results[email] = True
# Loop through first 20 pages of results
domain_ids = re.findall(r'<a href="/domain/([a-z0-9]+)\?p=', res['content'])
@@ -111,32 +96,19 @@ class sfp_skymem(SpiderFootPlugin):
if res['content'] is None:
break
- pat = re.compile("([a-zA-Z\.0-9_\-]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)")
- matches = re.findall(pat, res['content'])
- for match in matches:
- self.sf.debug("Found possible email: " + match)
-
- # Handle false positive matches
- if len(match) < 5:
- self.sf.debug("Likely invalid address.")
- continue
-
- # Handle messed up encodings
- if "%" in match:
- self.sf.debug("Skipped address: " + match)
- continue
-
+ emails = self.sf.parseEmails(res['content'])
+ for email in emails:
# Skip unrelated emails
- mailDom = match.lower().split('@')[1]
+ mailDom = email.lower().split('@')[1]
if not self.getTarget().matches(mailDom):
- self.sf.debug("Skipped address: " + match)
+ self.sf.debug("Skipped address: " + email)
continue
- self.sf.info("Found e-mail address: " + match)
- if match not in self.results:
- evt = SpiderFootEvent("EMAILADDR", match, self.__name__, event)
+ self.sf.info("Found e-mail address: " + email)
+ if email not in self.results:
+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
self.notifyListeners(evt)
- self.results[match] = True
+ self.results[email] = True
# Check if we're on the last page of results
max_page = 0
diff --git a/sflib.py b/sflib.py
index d4dec00b..da1e7f87 100644
--- a/sflib.py
+++ b/sflib.py
@@ -889,6 +889,34 @@ class SpiderFoot:
return returnArr
+ # Find all emails within the supplied content
+ # Returns an Array
+ def parseEmails(self, data):
+ emails = list()
+ matches = re.findall(r'([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)', str(data).decode('unicode-escape'))
+
+ for match in matches:
+ self.debug("Found possible email: " + match)
+
+ # Handle false positive matches
+ if len(match) < 5:
+ self.debug("Skipped likely invalid email address.")
+ continue
+
+ # Handle messed up encodings
+ if "%" in match:
+ self.debug("Skipped invalid email address: " + match)
+ continue
+
+ # Handle truncated emails
+ if "..." in match:
+ self.debug("Skipped incomplete e-mail address: " + match)
+ continue
+
+ emails.append(match)
+
+ return set(emails)
+
# Find all URLs within the supplied content. This does not fetch any URLs!
# A dictionary will be returned, where each link will have the keys
# 'source': The URL where the link was obtained from