SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Fix-ups for properly using the respective APIs.
smicallef622d2a32ca4b
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.pyindex bc98b824..ebcd7996 100644--- a/modules/sfp_bingsearch.py+++ b/modules/sfp_bingsearch.py@@ -24,14 +24,16 @@ class sfp_bingsearch(SpiderFootPlugin):# Option descriptionsoptdescs = {"pages": "Number of max bing results to request from the API.",- "api_key": "Bing API Key."+ "api_key": "Bing API Key for Bing search."}- results = list()+ results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -44,21 +46,32 @@ class sfp_bingsearch(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["LINKED_URL_INTERNAL", "SEARCH_ENGINE_WEB_CONTENT"]+ return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"]def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_bingsearch but did not set a Bing API key!", False)+ self.errorState = True+ return None+if eventData in self.results:self.sf.debug("Already did a search for " + eventData + ", skipping.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = True# Sites hosted on the domain- results = self.sf.bingIterate(+ res = self.sf.bingIterate(searchString="site:" + eventData,opts={"timeout": self.opts["_fetchtimeout"],@@ -67,15 +80,16 @@ class sfp_bingsearch(SpiderFootPlugin):"api_key": self.opts["api_key"],},)- if results is None:+ if res is None:# Failed to talk to the bing API or no results returnedreturn None- urls = results["urls"]- new_links = list(set(urls) - set(self.results))+ urls = res["urls"]+ new_links = list(set(urls) - set(self.results.keys()))# Add new links to results- self.results.extend(new_links)+ for l in new_links:+ self.results[l] = Trueinternal_links = [link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)@@ -87,20 +101,10 @@ class sfp_bingsearch(SpiderFootPlugin):self.notifyListeners(evt)if internal_links:- # Submit the bing results for analysis- bingsearch_url = results["webSearchUrl"]- response = self.sf.fetchUrl(- bingsearch_url,- timeout=self.opts["_fetchtimeout"],- useragent=self.opts["_useragent"],+ evt = SpiderFootEvent(+ "RAW_RIR_DATA", str(res), self.__name__, event)- if response['code'].startswith('2'):- evt = SpiderFootEvent(- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event- )- self.notifyListeners(evt)- else:- self.sf.error("Failed to fetch bing web search URL", exception=False)+ self.notifyListeners(evt)diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.pyindex ff5e6186..fc607448 100644--- a/modules/sfp_bingsharedip.py+++ b/modules/sfp_bingsharedip.py@@ -25,7 +25,7 @@ class sfp_bingsharedip(SpiderFootPlugin):"pages": 20,"verify": True,"maxcohost": 100,- "api_key": "",+ "api_key": ""}# Option descriptions@@ -34,17 +34,19 @@ class sfp_bingsharedip(SpiderFootPlugin):"pages": "Number of max bing results to request from API.","verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.","maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",- "api_key": "Bing API Key.",+ "api_key": "Bing API Key for shared IP search."}- results = dict()+ results = Nonecohostcount = 0+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = dict()+ self.results = self.tempStorage()self.cohostcount = 0self.__dataSource__ = "Bing"+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -57,7 +59,7 @@ class sfp_bingsharedip(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["CO_HOSTED_SITE", "IP_ADDRESS", "SEARCH_ENGINE_WEB_CONTENT"]+ return ["CO_HOSTED_SITE", "IP_ADDRESS", "RAW_RIR_DATA"]def validateIP(self, host, ip):try:@@ -83,8 +85,16 @@ class sfp_bingsharedip(SpiderFootPlugin):eventData = event.dataself.currentEventSrc = event+ if self.errorState:+ return None+self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)+ if self.opts['api_key'] == "" and self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_bingsharedip but did not set a Bing API key!", False)+ self.errorState = True+ return None+# Don't look up stuff twiceif eventData in self.results:self.sf.debug("Skipping " + eventData + " as already mapped.")@@ -115,7 +125,7 @@ class sfp_bingsharedip(SpiderFootPlugin):if self.checkForStop():return None- results = self.sf.bingIterate(+ res = self.sf.bingIterate(searchString="ip:" + ip,opts={"timeout": self.opts["_fetchtimeout"],@@ -124,11 +134,11 @@ class sfp_bingsharedip(SpiderFootPlugin):"api_key": self.opts["api_key"],},)- if results is None:+ if res is None:# Failed to talk to bing api or no results returnedreturn None- urls = results["urls"]+ urls = res["urls"]for url in urls:self.sf.info("Found something on same IP: " + url)@@ -163,23 +173,10 @@ class sfp_bingsharedip(SpiderFootPlugin):myres.append(site)if urls:- # Submit the bing results for analysis- bingsearch_url = results["webSearchUrl"]- response = self.sf.fetchUrl(- bingsearch_url,- timeout=self.opts["_fetchtimeout"],- useragent=self.opts["_useragent"],+ evt = SpiderFootEvent(+ "RAW_RIR_DATA", str(res), self.__name__, event)- if response['status'] == 'OK':- evt = SpiderFootEvent(- "SEARCH_ENGINE_WEB_CONTENT",- response["content"],- self.__name__,- event,- )- self.notifyListeners(evt)- else:- self.sf.error("Failed to fetch bing web search URL", exception=False)+ self.notifyListeners(evt)# End of sfp_bingsharedip classdiff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.pyindex a5c1de41..9d995f47 100644--- a/modules/sfp_googlesearch.py+++ b/modules/sfp_googlesearch.py@@ -27,16 +27,18 @@ class sfp_googlesearch(SpiderFootPlugin):# Option descriptionsoptdescs = {- "api_key": "Google API Key.",+ "api_key": "Google API Key for Google search.","cse_id": "Google Custom Search Engine ID."}# Target- results = list()+ results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -49,21 +51,31 @@ class sfp_googlesearch(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["LINKED_URL_INTERNAL", "SEARCH_ENGINE_WEB_CONTENT"]+ return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"]def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data+ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_googlesearch but did not set a Google API key!", False)+ self.errorState = True+ return None+if eventData in self.results:self.sf.debug("Already did a search for " + eventData + ", skipping.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = True# Sites hosted on the domain- results = self.sf.googleIterate(+ res = self.sf.googleIterate(searchString="site:" + eventData,opts={"timeout": self.opts["_fetchtimeout"],@@ -72,15 +84,16 @@ class sfp_googlesearch(SpiderFootPlugin):"cse_id": self.opts["cse_id"],},)- if results is None:+ if res is None:# Failed to talk to the Google API or no results returnedreturn None- urls = results["urls"]- new_links = list(set(urls) - set(self.results))+ urls = res["urls"]+ new_links = list(set(urls) - set(self.results.keys()))# Add new links to results- self.results.extend(new_links)+ for l in new_links:+ self.results[l] = Trueinternal_links = [link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)@@ -92,19 +105,9 @@ class sfp_googlesearch(SpiderFootPlugin):self.notifyListeners(evt)if internal_links:- # Submit the Google results for analysis- googlesearch_url = results["webSearchUrl"]- response = self.sf.fetchUrl(- googlesearch_url,- timeout=self.opts["_fetchtimeout"],- useragent=self.opts["_useragent"],+ evt = SpiderFootEvent(+ "RAW_RIR_DATA", str(res), self.__name__, event)- if response['status'] == 'OK':- evt = SpiderFootEvent(- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event- )- self.notifyListeners(evt)- else:- self.sf.error("Failed to fetch Google web search URL", exception=False)+ self.notifyListeners(evt)# End of sfp_googlesearch classdiff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.pyindex 2560a383..a9e28121 100644--- a/modules/sfp_onioncity.py+++ b/modules/sfp_onioncity.py@@ -27,17 +27,19 @@ class sfp_onioncity(SpiderFootPlugin):# Option descriptionsoptdescs = {- "api_key": "Google API Key.",+ "api_key": "Google API Key for Onion.link search.","cse_id": "Google Custom Search Engine ID.",'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target."}# Target- results = list()+ results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -51,21 +53,31 @@ class sfp_onioncity(SpiderFootPlugin):# produced.def producedEvents(self):return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT",- "SEARCH_ENGINE_WEB_CONTENT"]+ "RAW_RIR_DATA"]def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data+ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_onioncity but did not set a Google API key!", False)+ self.errorState = True+ return None+if eventData in self.results:self.sf.debug("Already did a search for " + eventData + ", skipping.")return Noneelse:- self.results.append(eventData)+ self.results[eventData] = True# Sites hosted on the domain- results = self.sf.googleIterate(+ res = self.sf.googleIterate(searchString="+site:onion.link " + eventData,opts={"timeout": self.opts["_fetchtimeout"],@@ -74,18 +86,19 @@ class sfp_onioncity(SpiderFootPlugin):"cse_id": self.opts["cse_id"],},)- if results is None:+ if res is None:# Failed to talk to the bing API or no results returnedreturn None- urls = results["urls"]- new_links = list(set(urls) - set(self.results))+ urls = res["urls"]+ new_links = list(set(urls) - set(self.results.keys()))# Add new links to results- self.results.extend(new_links)+ for l in new_links:+ self.results[l] = True# Submit the Google results for analysis- googlesearch_url = results["webSearchUrl"]+ googlesearch_url = res["webSearchUrl"]response = self.sf.fetchUrl(googlesearch_url,timeout=self.opts["_fetchtimeout"],@@ -93,7 +106,7 @@ class sfp_onioncity(SpiderFootPlugin):)if response['code'].startswith('2'):evt = SpiderFootEvent(- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event+ "RAW_RIR_DATA", response["content"], self.__name__, event)self.notifyListeners(evt)else:diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.pyindex 8e650478..c4c02521 100644--- a/modules/sfp_pastebin.py+++ b/modules/sfp_pastebin.py@@ -27,7 +27,7 @@ class sfp_pastebin(SpiderFootPlugin):# Option descriptionsoptdescs = {- "api_key": "Google API Key.",+ "api_key": "Google API Key for PasteBin search.","cse_id": "Google Custom Search Engine ID.",}@@ -35,11 +35,13 @@ class sfp_pastebin(SpiderFootPlugin):'pastebin': "pastebin.com"}- results = list()+ results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = list()+ self.results = self.tempStorage()+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -59,15 +61,23 @@ class sfp_pastebin(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.data+ if self.errorState:+ return None++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_pastebin but did not set a Google API key!", False)+ self.errorState = True+ return None+if eventData in self.results:return Noneelse:- self.results.append(eventData)+ self.results[eventData] = Truefor dom in self.domains.keys():links = list()target = self.domains[dom]- results = self.sf.googleIterate(+ res = self.sf.googleIterate(searchString="+site:{target_site} \"{search_keyword}\"".format(target_site=target,search_keyword=eventData,@@ -80,15 +90,16 @@ class sfp_pastebin(SpiderFootPlugin):},)- if results is None:+ if res is None:# Failed to talk to the Google API or no results returnedreturn None- urls = results["urls"]- new_links = list(set(urls) - set(self.results))+ urls = res["urls"]+ new_links = list(set(urls) - set(self.results.keys()))# Add new links to results- self.results.extend(new_links)+ for l in new_links:+ self.results[l] = Truerelevant_links = [link for link in new_links if self.sf.urlBaseUrl(link).endswith(target)diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.pyindex 675e8f34..0b326996 100644--- a/modules/sfp_socialprofiles.py+++ b/modules/sfp_socialprofiles.py@@ -61,18 +61,20 @@ class sfp_socialprofiles(SpiderFootPlugin):"count": "Number of bing search engine results of identified profiles to iterate through.","method": "Search engine to use: 'google' or 'bing'.","tighten": "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",- "bing_api_key": "Bing API Key.",- "google_api_key": "Google API Key.",+ "bing_api_key": "Bing API Key for social media profile search.",+ "google_api_key": "Google API Key for social media profile search.","google_cse_id": "Google Custom Search Engine ID.",}keywords = None- results = dict()+ results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfc- self.results = dict()+ self.results = self.tempStorage()self.keywords = None+ self.errorState = Falsefor opt in userOpts.keys():self.opts[opt] = userOpts[opt]@@ -85,7 +87,7 @@ class sfp_socialprofiles(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["SOCIAL_MEDIA", "SEARCH_ENGINE_WEB_CONTENT"]+ return ["SOCIAL_MEDIA", "RAW_RIR_DATA"]# Handle events sent to this moduledef handleEvent(self, event):@@ -94,8 +96,16 @@ class sfp_socialprofiles(SpiderFootPlugin):eventData = event.dataself.currentEventSrc = event+ if self.errorState:+ return None+self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)+ if self.opts['google_api_key'] == "" and self.opts['bing_api_key'] == "":+ self.sf.error("You enabled sfp_socialprofiles but did not set a Google or Bing API key!", False)+ self.errorState = True+ return None+# Don't look up stuff twiceif eventData in self.results:self.sf.debug("Skipping " + eventData + " as already mapped.")@@ -113,7 +123,7 @@ class sfp_socialprofiles(SpiderFootPlugin):for site in sites:s = unicode(sites[site][0]).format(name=eventData)searchStr = s.replace(" ", "%20")- results = None+ res = Noneif self.opts["method"].lower() == "yahoo":self.sf.error(@@ -123,7 +133,7 @@ class sfp_socialprofiles(SpiderFootPlugin):return Noneif self.opts["method"].lower() == "google":- results = self.sf.googleIterate(+ res = self.sf.googleIterate(searchString=searchStr,opts={"timeout": self.opts["_fetchtimeout"],@@ -135,7 +145,7 @@ class sfp_socialprofiles(SpiderFootPlugin):self.__dataSource__ = "Google"if self.opts["method"].lower() == "bing":- results = self.sf.bingIterate(+ res = self.sf.bingIterate(searchString=searchStr,opts={"timeout": self.opts["_fetchtimeout"],@@ -146,24 +156,23 @@ class sfp_socialprofiles(SpiderFootPlugin):)self.__dataSource__ = "Bing"- if results is None:+ if res is None:self.sf.info("No data returned from " + self.opts["method"] + ".")continueif self.checkForStop():return None-# Submit the results for analysisevt = SpiderFootEvent(- "SEARCH_ENGINE_WEB_CONTENT", str(results), self.__name__, event+ "RAW_RIR_DATA", str(res), self.__name__, event)self.notifyListeners(evt)instances = list()for searchDom in sites[site][1]:# Search both the urls & the search engine web content- search_string = " ".join(results["urls"] + [str(results)])+ search_string = " ".join(res["urls"] + [str(res)])matches = re.findall(searchDom, search_string, re.IGNORECASE | re.MULTILINE