Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

feat: update socialprofiles module

Amr Awadd8fba40aa378
diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.py
index 75f3ffec..87e6c970 100644
--- a/modules/sfp_socialprofiles.py
+++ b/modules/sfp_socialprofiles.py
@@ -17,36 +17,53 @@ import urllib2
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
sites = {
- # Search string to use, domain name the profile will sit on within
+ # Search string to use, domain name the profile will sit on within
# those search results.
- "Facebook": ['+title:%22{0}%22%20+site:facebook.com',
- ['"(https?://[a-z\.]*facebook.[a-z\.]+/[^/\"<> ]+)"',
- '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/\"<> ]+)']],
- "Google+": ['+title:%22{0}%22%20+site:plus.google.com',
- ['"(https?://plus.google.[a-z\.]+/\d+[^\"<>\/ ]+)"',
- '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/\"<> ]+)']],
- "LinkedIn": ['+title:%22{0}%22%20+site:linkedin.com',
- ['"(https?://[a-z\.]*linkedin.[a-z\.]+/[^/\"<> ]+)"',
- '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\/\"<> ]+)']]
+ "Facebook": [
+ "+title:%22{0}%22%20+site:facebook.com",
+ [
+ '"(https?://[a-z\.]*facebook.[a-z\.]+/[^/"<> ]+)"',
+ '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"<> ]+)',
+ ],
+ ],
+ "Google+": [
+ "+title:%22{0}%22%20+site:plus.google.com",
+ [
+ '"(https?://plus.google.[a-z\.]+/\d+[^"<>\/ ]+)"',
+ '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"<> ]+)',
+ ],
+ ],
+ "LinkedIn": [
+ "+title:%22{0}%22%20+site:linkedin.com",
+ [
+ '"(https?://[a-z\.]*linkedin.[a-z\.]+/[^/"<> ]+)"',
+ '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\/"<> ]+)',
+ ],
+ ],
}
class sfp_socialprofiles(SpiderFootPlugin):
- """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Tries to discover the social media profiles for human names identified."""
-
+ """Social Media Profiles:Footprint,Passive:Social Media:slow,apikey:Tries to discover the social media profiles for human names identified."""
# Default options
opts = {
- 'pages': 1,
- 'method': "bing",
- 'tighten': True
+ "count": 20,
+ "method": "bing",
+ "tighten": True,
+ "bing_api_key": "",
+ "google_api_key": "",
+ "google_cse_id": "",
}
# Option descriptions
optdescs = {
- 'pages': "Number of search engine pages of identified profiles to iterate through.",
- 'tighten': "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",
- 'method': "Search engine to use: google or bing."
+ "count": "Number of bing search engine results of identified profiles to iterate through.",
+ "method": "Search engine to use: google or bing.",
+ "tighten": "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",
+ "bing_api_key": "Bing API Key.",
+ "google_api_key": "Google API Key.",
+ "google_cse_id": "Google Custom Search Engine ID.",
}
keywords = None
@@ -87,8 +104,9 @@ class sfp_socialprofiles(SpiderFootPlugin):
self.results[eventData] = True
if self.keywords is None:
- self.keywords = self.sf.domainKeywords(self.getTarget().getNames(),
- self.opts['_internettlds'])
+ self.keywords = self.sf.domainKeywords(
+ self.getTarget().getNames(), self.opts["_internettlds"]
+ )
if len(self.keywords) == 0:
self.keywords = None
@@ -97,77 +115,113 @@ class sfp_socialprofiles(SpiderFootPlugin):
searchStr = s.replace(" ", "%20")
results = None
- if self.opts['method'].lower() == "yahoo":
- self.sf.error("Yahoo is no longer supported. Please try 'bing' or 'google'.", False)
+ if self.opts["method"].lower() == "yahoo":
+ self.sf.error(
+ "Yahoo is no longer supported. Please try 'bing' or 'google'.",
+ False,
+ )
return None
- if self.opts['method'].lower() == "google":
- results = self.sf.googleIterate(searchStr, dict(limit=self.opts['pages'],
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
+ if self.opts["method"].lower() == "google":
+ results = self.sf.googleIterate(
+ searchString=searchStr,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
+ "api_key": self.opts["google_api_key"],
+ "cse_id": self.opts["google_cse_id"],
+ },
+ )
self.__dataSource__ = "Google"
- if self.opts['method'].lower() == "bing":
- results = self.sf.bingIterate(searchStr, dict(limit=self.opts['pages'],
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
+ if self.opts["method"].lower() == "bing":
+ results = self.sf.bingIterate(
+ searchString=searchStr,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
+ "count": self.opts["count"],
+ "api_key": self.opts["bing_api_key"],
+ },
+ )
self.__dataSource__ = "Bing"
if results is None:
- self.sf.info("No data returned from " + self.opts['method'] + ".")
+ self.sf.info("No data returned from " + self.opts["method"] + ".")
return None
if self.checkForStop():
return None
- for key in results:
- instances = list()
+ web_search_url = results["webSearchUrl"]
+ response = self.sf.fetchUrl(
+ web_search_url,
+ timeout=self.opts["_fetchtimeout"],
+ useragent=self.opts["_useragent"],
+ )
+
+ if response["status"] != "OK":
+ self.sf.error("Failed to fetch bing web search URL", exception=False)
+ return None
+
+ # Submit the results for analysis
+ evt = SpiderFootEvent(
+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
+ )
+ self.notifyListeners(evt)
+
+ for searchDom in sites[site][1]:
+ matches = re.findall(
+ searchDom, response["content"], re.IGNORECASE | re.MULTILINE
+ )
- for searchDom in sites[site][1]:
- matches = re.findall(searchDom, results[key], re.IGNORECASE|re.MULTILINE)
+ if not matches:
+ continue
- if not matches:
+ for match in matches:
+ self.sf.debug("Match found: " + match)
+ if match in instances:
continue
+ else:
+ instances.append(match)
+
+ if self.checkForStop():
+ return None
- for match in matches:
- self.sf.debug("Match found: " + match)
- if match in instances:
+ # Fetch the profile page if we are checking
+ # for a firm relationship.
+ # Keywords might be empty if the target was an IP, subnet or name.
+ if self.opts["tighten"] and self.keywords:
+ match = urllib2.unquote(match)
+ self.sf.debug(
+ "Tightening results to look for " + str(self.keywords)
+ )
+ pres = self.sf.fetchUrl(
+ match,
+ timeout=self.opts["_fetchtimeout"],
+ useragent=self.opts["_useragent"],
+ )
+
+ if pres["content"] is None:
continue
else:
- instances.append(match)
-
- if self.checkForStop():
- return None
+ found = False
+ for kw in self.keywords:
+ if re.search(
+ "[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",
+ pres["content"],
+ re.IGNORECASE,
+ ):
+ found = True
+ if not found:
+ continue
- # Fetch the profile page if we are checking
- # for a firm relationship.
- # Keywords might be empty if the target was an IP, subnet or name.
- if self.opts['tighten'] and self.keywords:
- match = urllib2.unquote(match)
- self.sf.debug("Tightening results to look for " + str(self.keywords))
- pres = self.sf.fetchUrl(match, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
+ self.sf.info("Social Media Profile found at " + site + ": " + match)
+ match = urllib2.unquote(match)
+ evt = SpiderFootEvent(
+ "SOCIAL_MEDIA", site + ": " + match, self.__name__, event
+ )
+ self.notifyListeners(evt)
- if pres['content'] is None:
- continue
- else:
- found = False
- for kw in self.keywords:
- if re.search("[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",
- pres['content'], re.IGNORECASE):
- found = True
- if not found:
- continue
-
- self.sf.info("Social Media Profile found at " + site + ": " + match)
- match = urllib2.unquote(match)
- evt = SpiderFootEvent("SOCIAL_MEDIA", site + ": " + match,
- self.__name__, event)
- self.notifyListeners(evt)
-
- # Submit the results for analysis
- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", results[key],
- self.__name__, event)
- self.notifyListeners(evt)
# End of sfp_socialprofiles class