SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
feat: update socialprofiles module
Amr Awadd8fba40aa378
diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.pyindex 75f3ffec..87e6c970 100644--- a/modules/sfp_socialprofiles.py+++ b/modules/sfp_socialprofiles.py@@ -17,36 +17,53 @@ import urllib2from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventsites = {- # Search string to use, domain name the profile will sit on within+ # Search string to use, domain name the profile will sit on within# those search results.- "Facebook": ['+title:%22{0}%22%20+site:facebook.com',- ['"(https?://[a-z\.]*facebook.[a-z\.]+/[^/\"<> ]+)"',- '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/\"<> ]+)']],- "Google+": ['+title:%22{0}%22%20+site:plus.google.com',- ['"(https?://plus.google.[a-z\.]+/\d+[^\"<>\/ ]+)"',- '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/\"<> ]+)']],- "LinkedIn": ['+title:%22{0}%22%20+site:linkedin.com',- ['"(https?://[a-z\.]*linkedin.[a-z\.]+/[^/\"<> ]+)"',- '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\/\"<> ]+)']]+ "Facebook": [+ "+title:%22{0}%22%20+site:facebook.com",+ [+ '"(https?://[a-z\.]*facebook.[a-z\.]+/[^/"<> ]+)"',+ '(https?%3a%2f%2f[a-z\.]*facebook.[a-z\.]+%2f[^\/"<> ]+)',+ ],+ ],+ "Google+": [+ "+title:%22{0}%22%20+site:plus.google.com",+ [+ '"(https?://plus.google.[a-z\.]+/\d+[^"<>\/ ]+)"',+ '(https?%3a%2f%2fplus.google.[a-z\.]+%2f\d+[^\/"<> ]+)',+ ],+ ],+ "LinkedIn": [+ "+title:%22{0}%22%20+site:linkedin.com",+ [+ '"(https?://[a-z\.]*linkedin.[a-z\.]+/[^/"<> ]+)"',+ '(https?%3a%2f%2f[a-z\.]*linkedin.[a-z\.]+%2f[^\/"<> ]+)',+ ],+ ],}class sfp_socialprofiles(SpiderFootPlugin):- """Social Media Profiles:Footprint,Passive:Social Media:slow,errorprone:Tries to discover the social media profiles for human names identified."""-+ """Social Media Profiles:Footprint,Passive:Social Media:slow,apikey:Tries to discover the social media profiles for human names identified."""# Default optionsopts = {- 'pages': 1,- 'method': "bing",- 'tighten': True+ "count": 20,+ "method": "bing",+ "tighten": True,+ "bing_api_key": "",+ "google_api_key": "",+ "google_cse_id": "",}# Option descriptionsoptdescs = {- 'pages': "Number of search engine pages of identified profiles to iterate through.",- 'tighten': "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",- 'method': "Search engine to use: google or bing."+ "count": "Number of bing search engine results of identified profiles to iterate through.",+ "method": "Search engine to use: google or bing.",+ "tighten": "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",+ "bing_api_key": "Bing API Key.",+ "google_api_key": "Google API Key.",+ "google_cse_id": "Google Custom Search Engine ID.",}keywords = None@@ -87,8 +104,9 @@ class sfp_socialprofiles(SpiderFootPlugin):self.results[eventData] = Trueif self.keywords is None:- self.keywords = self.sf.domainKeywords(self.getTarget().getNames(),- self.opts['_internettlds'])+ self.keywords = self.sf.domainKeywords(+ self.getTarget().getNames(), self.opts["_internettlds"]+ )if len(self.keywords) == 0:self.keywords = None@@ -97,77 +115,113 @@ class sfp_socialprofiles(SpiderFootPlugin):searchStr = s.replace(" ", "%20")results = None- if self.opts['method'].lower() == "yahoo":- self.sf.error("Yahoo is no longer supported. Please try 'bing' or 'google'.", False)+ if self.opts["method"].lower() == "yahoo":+ self.sf.error(+ "Yahoo is no longer supported. Please try 'bing' or 'google'.",+ False,+ )return None- if self.opts['method'].lower() == "google":- results = self.sf.googleIterate(searchStr, dict(limit=self.opts['pages'],- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))+ if self.opts["method"].lower() == "google":+ results = self.sf.googleIterate(+ searchString=searchStr,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],+ "api_key": self.opts["google_api_key"],+ "cse_id": self.opts["google_cse_id"],+ },+ )self.__dataSource__ = "Google"- if self.opts['method'].lower() == "bing":- results = self.sf.bingIterate(searchStr, dict(limit=self.opts['pages'],- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))+ if self.opts["method"].lower() == "bing":+ results = self.sf.bingIterate(+ searchString=searchStr,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],+ "count": self.opts["count"],+ "api_key": self.opts["bing_api_key"],+ },+ )self.__dataSource__ = "Bing"if results is None:- self.sf.info("No data returned from " + self.opts['method'] + ".")+ self.sf.info("No data returned from " + self.opts["method"] + ".")return Noneif self.checkForStop():return None- for key in results:- instances = list()+ web_search_url = results["webSearchUrl"]+ response = self.sf.fetchUrl(+ web_search_url,+ timeout=self.opts["_fetchtimeout"],+ useragent=self.opts["_useragent"],+ )++ if response["status"] != "OK":+ self.sf.error("Failed to fetch bing web search URL", exception=False)+ return None++ # Submit the results for analysis+ evt = SpiderFootEvent(+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event+ )+ self.notifyListeners(evt)++ for searchDom in sites[site][1]:+ matches = re.findall(+ searchDom, response["content"], re.IGNORECASE | re.MULTILINE+ )- for searchDom in sites[site][1]:- matches = re.findall(searchDom, results[key], re.IGNORECASE|re.MULTILINE)+ if not matches:+ continue- if not matches:+ for match in matches:+ self.sf.debug("Match found: " + match)+ if match in instances:continue+ else:+ instances.append(match)++ if self.checkForStop():+ return None- for match in matches:- self.sf.debug("Match found: " + match)- if match in instances:+ # Fetch the profile page if we are checking+ # for a firm relationship.+ # Keywords might be empty if the target was an IP, subnet or name.+ if self.opts["tighten"] and self.keywords:+ match = urllib2.unquote(match)+ self.sf.debug(+ "Tightening results to look for " + str(self.keywords)+ )+ pres = self.sf.fetchUrl(+ match,+ timeout=self.opts["_fetchtimeout"],+ useragent=self.opts["_useragent"],+ )++ if pres["content"] is None:continueelse:- instances.append(match)-- if self.checkForStop():- return None+ found = False+ for kw in self.keywords:+ if re.search(+ "[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",+ pres["content"],+ re.IGNORECASE,+ ):+ found = True+ if not found:+ continue- # Fetch the profile page if we are checking- # for a firm relationship.- # Keywords might be empty if the target was an IP, subnet or name.- if self.opts['tighten'] and self.keywords:- match = urllib2.unquote(match)- self.sf.debug("Tightening results to look for " + str(self.keywords))- pres = self.sf.fetchUrl(match, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ self.sf.info("Social Media Profile found at " + site + ": " + match)+ match = urllib2.unquote(match)+ evt = SpiderFootEvent(+ "SOCIAL_MEDIA", site + ": " + match, self.__name__, event+ )+ self.notifyListeners(evt)- if pres['content'] is None:- continue- else:- found = False- for kw in self.keywords:- if re.search("[^a-zA-Z\-\_]" + kw + "[^a-zA-Z\-\_]",- pres['content'], re.IGNORECASE):- found = True- if not found:- continue-- self.sf.info("Social Media Profile found at " + site + ": " + match)- match = urllib2.unquote(match)- evt = SpiderFootEvent("SOCIAL_MEDIA", site + ": " + match,- self.__name__, event)- self.notifyListeners(evt)-- # Submit the results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", results[key],- self.__name__, event)- self.notifyListeners(evt)# End of sfp_socialprofiles class