Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Fix-ups for properly using the respective APIs.

smicallef622d2a32ca4b
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.py
index bc98b824..ebcd7996 100644
--- a/modules/sfp_bingsearch.py
+++ b/modules/sfp_bingsearch.py
@@ -24,14 +24,16 @@ class sfp_bingsearch(SpiderFootPlugin):
# Option descriptions
optdescs = {
"pages": "Number of max bing results to request from the API.",
- "api_key": "Bing API Key."
+ "api_key": "Bing API Key for Bing search."
}
- results = list()
+ results = None
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -44,21 +46,32 @@ class sfp_bingsearch(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["LINKED_URL_INTERNAL", "SEARCH_ENGINE_WEB_CONTENT"]
+ return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"]
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
+
+ if self.errorState:
+ return None
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if self.opts['api_key'] == "":
+ self.sf.error("You enabled sfp_bingsearch but did not set a Bing API key!", False)
+ self.errorState = True
+ return None
+
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
# Sites hosted on the domain
- results = self.sf.bingIterate(
+ res = self.sf.bingIterate(
searchString="site:" + eventData,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -67,15 +80,16 @@ class sfp_bingsearch(SpiderFootPlugin):
"api_key": self.opts["api_key"],
},
)
- if results is None:
+ if res is None:
# Failed to talk to the bing API or no results returned
return None
- urls = results["urls"]
- new_links = list(set(urls) - set(self.results))
+ urls = res["urls"]
+ new_links = list(set(urls) - set(self.results.keys()))
# Add new links to results
- self.results.extend(new_links)
+ for l in new_links:
+ self.results[l] = True
internal_links = [
link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)
@@ -87,20 +101,10 @@ class sfp_bingsearch(SpiderFootPlugin):
self.notifyListeners(evt)
if internal_links:
- # Submit the bing results for analysis
- bingsearch_url = results["webSearchUrl"]
- response = self.sf.fetchUrl(
- bingsearch_url,
- timeout=self.opts["_fetchtimeout"],
- useragent=self.opts["_useragent"],
+ evt = SpiderFootEvent(
+ "RAW_RIR_DATA", str(res), self.__name__, event
)
- if response['code'].startswith('2'):
- evt = SpiderFootEvent(
- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
- )
- self.notifyListeners(evt)
- else:
- self.sf.error("Failed to fetch bing web search URL", exception=False)
+ self.notifyListeners(evt)
diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.py
index ff5e6186..fc607448 100644
--- a/modules/sfp_bingsharedip.py
+++ b/modules/sfp_bingsharedip.py
@@ -25,7 +25,7 @@ class sfp_bingsharedip(SpiderFootPlugin):
"pages": 20,
"verify": True,
"maxcohost": 100,
- "api_key": "",
+ "api_key": ""
}
# Option descriptions
@@ -34,17 +34,19 @@ class sfp_bingsharedip(SpiderFootPlugin):
"pages": "Number of max bing results to request from API.",
"verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.",
"maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",
- "api_key": "Bing API Key.",
+ "api_key": "Bing API Key for shared IP search."
}
- results = dict()
+ results = None
cohostcount = 0
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = dict()
+ self.results = self.tempStorage()
self.cohostcount = 0
self.__dataSource__ = "Bing"
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -57,7 +59,7 @@ class sfp_bingsharedip(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["CO_HOSTED_SITE", "IP_ADDRESS", "SEARCH_ENGINE_WEB_CONTENT"]
+ return ["CO_HOSTED_SITE", "IP_ADDRESS", "RAW_RIR_DATA"]
def validateIP(self, host, ip):
try:
@@ -83,8 +85,16 @@ class sfp_bingsharedip(SpiderFootPlugin):
eventData = event.data
self.currentEventSrc = event
+ if self.errorState:
+ return None
+
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+ if self.opts['api_key'] == "" and self.opts['api_key'] == "":
+ self.sf.error("You enabled sfp_bingsharedip but did not set a Bing API key!", False)
+ self.errorState = True
+ return None
+
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug("Skipping " + eventData + " as already mapped.")
@@ -115,7 +125,7 @@ class sfp_bingsharedip(SpiderFootPlugin):
if self.checkForStop():
return None
- results = self.sf.bingIterate(
+ res = self.sf.bingIterate(
searchString="ip:" + ip,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -124,11 +134,11 @@ class sfp_bingsharedip(SpiderFootPlugin):
"api_key": self.opts["api_key"],
},
)
- if results is None:
+ if res is None:
# Failed to talk to bing api or no results returned
return None
- urls = results["urls"]
+ urls = res["urls"]
for url in urls:
self.sf.info("Found something on same IP: " + url)
@@ -163,23 +173,10 @@ class sfp_bingsharedip(SpiderFootPlugin):
myres.append(site)
if urls:
- # Submit the bing results for analysis
- bingsearch_url = results["webSearchUrl"]
- response = self.sf.fetchUrl(
- bingsearch_url,
- timeout=self.opts["_fetchtimeout"],
- useragent=self.opts["_useragent"],
+ evt = SpiderFootEvent(
+ "RAW_RIR_DATA", str(res), self.__name__, event
)
- if response['status'] == 'OK':
- evt = SpiderFootEvent(
- "SEARCH_ENGINE_WEB_CONTENT",
- response["content"],
- self.__name__,
- event,
- )
- self.notifyListeners(evt)
- else:
- self.sf.error("Failed to fetch bing web search URL", exception=False)
+ self.notifyListeners(evt)
# End of sfp_bingsharedip class
diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.py
index a5c1de41..9d995f47 100644
--- a/modules/sfp_googlesearch.py
+++ b/modules/sfp_googlesearch.py
@@ -27,16 +27,18 @@ class sfp_googlesearch(SpiderFootPlugin):
# Option descriptions
optdescs = {
- "api_key": "Google API Key.",
+ "api_key": "Google API Key for Google search.",
"cse_id": "Google Custom Search Engine ID."
}
# Target
- results = list()
+ results = None
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -49,21 +51,31 @@ class sfp_googlesearch(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["LINKED_URL_INTERNAL", "SEARCH_ENGINE_WEB_CONTENT"]
+ return ["LINKED_URL_INTERNAL", "RAW_RIR_DATA"]
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
+ if self.errorState:
+ return None
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if self.opts['api_key'] == "":
+ self.sf.error("You enabled sfp_googlesearch but did not set a Google API key!", False)
+ self.errorState = True
+ return None
+
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
# Sites hosted on the domain
- results = self.sf.googleIterate(
+ res = self.sf.googleIterate(
searchString="site:" + eventData,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -72,15 +84,16 @@ class sfp_googlesearch(SpiderFootPlugin):
"cse_id": self.opts["cse_id"],
},
)
- if results is None:
+ if res is None:
# Failed to talk to the Google API or no results returned
return None
- urls = results["urls"]
- new_links = list(set(urls) - set(self.results))
+ urls = res["urls"]
+ new_links = list(set(urls) - set(self.results.keys()))
# Add new links to results
- self.results.extend(new_links)
+ for l in new_links:
+ self.results[l] = True
internal_links = [
link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)
@@ -92,19 +105,9 @@ class sfp_googlesearch(SpiderFootPlugin):
self.notifyListeners(evt)
if internal_links:
- # Submit the Google results for analysis
- googlesearch_url = results["webSearchUrl"]
- response = self.sf.fetchUrl(
- googlesearch_url,
- timeout=self.opts["_fetchtimeout"],
- useragent=self.opts["_useragent"],
+ evt = SpiderFootEvent(
+ "RAW_RIR_DATA", str(res), self.__name__, event
)
- if response['status'] == 'OK':
- evt = SpiderFootEvent(
- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
- )
- self.notifyListeners(evt)
- else:
- self.sf.error("Failed to fetch Google web search URL", exception=False)
+ self.notifyListeners(evt)
# End of sfp_googlesearch class
diff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.py
index 2560a383..a9e28121 100644
--- a/modules/sfp_onioncity.py
+++ b/modules/sfp_onioncity.py
@@ -27,17 +27,19 @@ class sfp_onioncity(SpiderFootPlugin):
# Option descriptions
optdescs = {
- "api_key": "Google API Key.",
+ "api_key": "Google API Key for Onion.link search.",
"cse_id": "Google Custom Search Engine ID.",
'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target."
}
# Target
- results = list()
+ results = None
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -51,21 +53,31 @@ class sfp_onioncity(SpiderFootPlugin):
# produced.
def producedEvents(self):
return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT",
- "SEARCH_ENGINE_WEB_CONTENT"]
+ "RAW_RIR_DATA"]
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
+ if self.errorState:
+ return None
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if self.opts['api_key'] == "":
+ self.sf.error("You enabled sfp_onioncity but did not set a Google API key!", False)
+ self.errorState = True
+ return None
+
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
# Sites hosted on the domain
- results = self.sf.googleIterate(
+ res = self.sf.googleIterate(
searchString="+site:onion.link " + eventData,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -74,18 +86,19 @@ class sfp_onioncity(SpiderFootPlugin):
"cse_id": self.opts["cse_id"],
},
)
- if results is None:
+ if res is None:
# Failed to talk to the bing API or no results returned
return None
- urls = results["urls"]
- new_links = list(set(urls) - set(self.results))
+ urls = res["urls"]
+ new_links = list(set(urls) - set(self.results.keys()))
# Add new links to results
- self.results.extend(new_links)
+ for l in new_links:
+ self.results[l] = True
# Submit the Google results for analysis
- googlesearch_url = results["webSearchUrl"]
+ googlesearch_url = res["webSearchUrl"]
response = self.sf.fetchUrl(
googlesearch_url,
timeout=self.opts["_fetchtimeout"],
@@ -93,7 +106,7 @@ class sfp_onioncity(SpiderFootPlugin):
)
if response['code'].startswith('2'):
evt = SpiderFootEvent(
- "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
+ "RAW_RIR_DATA", response["content"], self.__name__, event
)
self.notifyListeners(evt)
else:
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.py
index 8e650478..c4c02521 100644
--- a/modules/sfp_pastebin.py
+++ b/modules/sfp_pastebin.py
@@ -27,7 +27,7 @@ class sfp_pastebin(SpiderFootPlugin):
# Option descriptions
optdescs = {
- "api_key": "Google API Key.",
+ "api_key": "Google API Key for PasteBin search.",
"cse_id": "Google Custom Search Engine ID.",
}
@@ -35,11 +35,13 @@ class sfp_pastebin(SpiderFootPlugin):
'pastebin': "pastebin.com"
}
- results = list()
+ results = None
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -59,15 +61,23 @@ class sfp_pastebin(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
+ if self.errorState:
+ return None
+
+ if self.opts['api_key'] == "":
+ self.sf.error("You enabled sfp_pastebin but did not set a Google API key!", False)
+ self.errorState = True
+ return None
+
if eventData in self.results:
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
for dom in self.domains.keys():
links = list()
target = self.domains[dom]
- results = self.sf.googleIterate(
+ res = self.sf.googleIterate(
searchString="+site:{target_site} \"{search_keyword}\"".format(
target_site=target,
search_keyword=eventData,
@@ -80,15 +90,16 @@ class sfp_pastebin(SpiderFootPlugin):
},
)
- if results is None:
+ if res is None:
# Failed to talk to the Google API or no results returned
return None
- urls = results["urls"]
- new_links = list(set(urls) - set(self.results))
+ urls = res["urls"]
+ new_links = list(set(urls) - set(self.results.keys()))
# Add new links to results
- self.results.extend(new_links)
+ for l in new_links:
+ self.results[l] = True
relevant_links = [
link for link in new_links if self.sf.urlBaseUrl(link).endswith(target)
diff --git a/modules/sfp_socialprofiles.py b/modules/sfp_socialprofiles.py
index 675e8f34..0b326996 100644
--- a/modules/sfp_socialprofiles.py
+++ b/modules/sfp_socialprofiles.py
@@ -61,18 +61,20 @@ class sfp_socialprofiles(SpiderFootPlugin):
"count": "Number of bing search engine results of identified profiles to iterate through.",
"method": "Search engine to use: 'google' or 'bing'.",
"tighten": "Tighten results by expecting to find the keyword of the target domain mentioned in the social media profile page results?",
- "bing_api_key": "Bing API Key.",
- "google_api_key": "Google API Key.",
+ "bing_api_key": "Bing API Key for social media profile search.",
+ "google_api_key": "Google API Key for social media profile search.",
"google_cse_id": "Google Custom Search Engine ID.",
}
keywords = None
- results = dict()
+ results = None
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = dict()
+ self.results = self.tempStorage()
self.keywords = None
+ self.errorState = False
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -85,7 +87,7 @@ class sfp_socialprofiles(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["SOCIAL_MEDIA", "SEARCH_ENGINE_WEB_CONTENT"]
+ return ["SOCIAL_MEDIA", "RAW_RIR_DATA"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -94,8 +96,16 @@ class sfp_socialprofiles(SpiderFootPlugin):
eventData = event.data
self.currentEventSrc = event
+ if self.errorState:
+ return None
+
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+ if self.opts['google_api_key'] == "" and self.opts['bing_api_key'] == "":
+ self.sf.error("You enabled sfp_socialprofiles but did not set a Google or Bing API key!", False)
+ self.errorState = True
+ return None
+
# Don't look up stuff twice
if eventData in self.results:
self.sf.debug("Skipping " + eventData + " as already mapped.")
@@ -113,7 +123,7 @@ class sfp_socialprofiles(SpiderFootPlugin):
for site in sites:
s = unicode(sites[site][0]).format(name=eventData)
searchStr = s.replace(" ", "%20")
- results = None
+ res = None
if self.opts["method"].lower() == "yahoo":
self.sf.error(
@@ -123,7 +133,7 @@ class sfp_socialprofiles(SpiderFootPlugin):
return None
if self.opts["method"].lower() == "google":
- results = self.sf.googleIterate(
+ res = self.sf.googleIterate(
searchString=searchStr,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -135,7 +145,7 @@ class sfp_socialprofiles(SpiderFootPlugin):
self.__dataSource__ = "Google"
if self.opts["method"].lower() == "bing":
- results = self.sf.bingIterate(
+ res = self.sf.bingIterate(
searchString=searchStr,
opts={
"timeout": self.opts["_fetchtimeout"],
@@ -146,24 +156,23 @@ class sfp_socialprofiles(SpiderFootPlugin):
)
self.__dataSource__ = "Bing"
- if results is None:
+ if res is None:
self.sf.info("No data returned from " + self.opts["method"] + ".")
continue
if self.checkForStop():
return None
-
# Submit the results for analysis
evt = SpiderFootEvent(
- "SEARCH_ENGINE_WEB_CONTENT", str(results), self.__name__, event
+ "RAW_RIR_DATA", str(res), self.__name__, event
)
self.notifyListeners(evt)
instances = list()
for searchDom in sites[site][1]:
# Search both the urls & the search engine web content
- search_string = " ".join(results["urls"] + [str(results)])
+ search_string = " ".join(res["urls"] + [str(res)])
matches = re.findall(
searchDom, search_string, re.IGNORECASE | re.MULTILINE