Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

feat: update module onioncity

Amr Awad5fbcaf2ed0f0
diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.py
index 7d33b1ea..83ad7146 100644
--- a/modules/sfp_googlesearch.py
+++ b/modules/sfp_googlesearch.py
@@ -72,7 +72,7 @@ class sfp_googlesearch(SpiderFootPlugin):
},
)
if results is None:
- # Failed to talk to the bing API or no results returned
+ # Failed to talk to the Google API or no results returned
return None
urls = results["urls"]
@@ -91,10 +91,10 @@ class sfp_googlesearch(SpiderFootPlugin):
self.notifyListeners(evt)
if internal_links:
- # Submit the bing results for analysis
- bingsearch_url = results["webSearchUrl"]
+ # Submit the Google results for analysis
+ googlesearch_url = results["webSearchUrl"]
response = self.sf.fetchUrl(
- bingsearch_url,
+ googlesearch_url,
timeout=self.opts["_fetchtimeout"],
useragent=self.opts["_useragent"],
)
@@ -104,6 +104,6 @@ class sfp_googlesearch(SpiderFootPlugin):
)
self.notifyListeners(evt)
else:
- self.sf.error("Failed to fetch bing web search URL", exception=False)
+ self.sf.error("Failed to fetch Google web search URL", exception=False)
# End of sfp_googlesearch class
diff --git a/modules/sfp_googlesearchdomain.py b/modules/sfp_googlesearchdomain.py
index 0216009e..7adf0521 100644
--- a/modules/sfp_googlesearchdomain.py
+++ b/modules/sfp_googlesearchdomain.py
@@ -85,7 +85,7 @@ class sfp_googlesearchdomain(SpiderFootPlugin):
},
)
if results is None:
- # Failed to talk to the bing API or no results returned
+ # Failed to talk to the Google API or no results returned
return None
urls = results["urls"]
@@ -104,10 +104,10 @@ class sfp_googlesearchdomain(SpiderFootPlugin):
self.notifyListeners(evt)
if internal_links:
- # Submit the bing results for analysis
- bingsearch_url = results["webSearchUrl"]
+ # Submit the Google results for analysis
+ googlesearch_url = results["webSearchUrl"]
response = self.sf.fetchUrl(
- bingsearch_url,
+ googlesearch_url,
timeout=self.opts["_fetchtimeout"],
useragent=self.opts["_useragent"],
)
@@ -117,6 +117,6 @@ class sfp_googlesearchdomain(SpiderFootPlugin):
)
self.notifyListeners(evt)
else:
- self.sf.error("Failed to fetch bing web search URL", exception=False)
+ self.sf.error("Failed to fetch Google web search URL", exception=False)
# End of sfp_googlesearchdomain class
diff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.py
index e1e6f38b..0837f9b6 100644
--- a/modules/sfp_onioncity.py
+++ b/modules/sfp_onioncity.py
@@ -15,18 +15,18 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
import re
class sfp_onioncity(SpiderFootPlugin):
- """Onion.link:Footprint,Investigate:Search Engines:errorprone:Search Tor 'Onion City' search engine for mentions of the target domain."""
+ """Onion.link:Footprint,Investigate:Search Engines:apikey:Search Tor 'Onion City' search engine for mentions of the target domain."""
# Default options
opts = {
- 'pages': 20, # Number of results pages to iterate
- 'fetchlinks': True
+ "api_key": "", "cse_id": "", 'fetchlinks': True
}
# Option descriptions
optdescs = {
- 'pages': "Number of results pages to iterate through.",
+ "api_key": "Google API Key.",
+ "cse_id": "Google Custom Search Engine ID.",
'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target."
}
@@ -63,73 +63,82 @@ class sfp_onioncity(SpiderFootPlugin):
self.results.append(eventData)
# Sites hosted on the domain
- pages = self.sf.googleIterate("site:onion.link+" + eventData,
- dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
- if pages is None:
- self.sf.info("No results returned from Google.")
+ results = self.sf.googleIterate(
+ searchString="site:onion.link+" + eventData,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
+ "api_key": self.opts["api_key"],
+ "cse_id": self.opts["cse_id"],
+ },
+ )
+ if results is None:
+ # Failed to talk to the bing API or no results returned
return None
- for page in pages.keys():
- if page in self.results:
- continue
- else:
- self.results.append(page)
+ urls = results["urls"]
+ new_links = list(set(urls) - set(self.results))
+
+ # Add new links to results
+ self.results.extend(new_links)
+
+ # Submit the Google results for analysis
+ googlesearch_url = results["webSearchUrl"]
+ response = self.sf.fetchUrl(
+ googlesearch_url,
+ timeout=self.opts["_fetchtimeout"],
+ useragent=self.opts["_useragent"],
+ )
+ if response['status'] == 'OK':
+ evt = SpiderFootEvent(
+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
+ )
+ self.notifyListeners(evt)
+ else:
+ self.sf.error("Failed to fetch Google web search URL", exception=False)
- # Check if we've been asked to stop
- if self.checkForStop():
- return None
+ # Check if we've been asked to stop
+ if self.checkForStop():
+ return None
- # Submit the google results for analysis
- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],
- self.__name__, event)
- self.notifyListeners(evt)
+ darknet_links = [
+ link for link in new_links if self.sf.urlFQDN(link).endswith(".onion.link")
+ ]
+
+ for link in darknet_links:
+ self.sf.debug("Found a darknet mention: " + link)
+ torlink = link.replace(".onion.link", ".onion")
+ if self.opts['fetchlinks']:
+ res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
- links = self.sf.parseLinks(page, pages[page], eventData)
- if len(links) == 0:
- continue
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
+ continue
+
+ # Sometimes onion city search results false positives
+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
+ continue
- for link in links:
- if link in self.results:
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)
+ self.notifyListeners(evt)
+
+ try:
+ startIndex = res['content'].index(eventData) - 120
+ endIndex = startIndex + len(eventData) + 240
+ except BaseException as e:
+ self.sf.debug("String not found in content.")
continue
- else:
- self.results.append(link)
- self.sf.debug("Found a darknet mention: " + link)
- if self.sf.urlFQDN(link).endswith(".onion.link"):
- if self.checkForStop():
- return None
- torlink = link.replace(".onion.link", ".onion")
- if self.opts['fetchlinks']:
- res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
-
- if res['content'] is None:
- self.sf.debug("Ignoring " + link + " as no data returned")
- continue
-
- # Sometimes onion city search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
- self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
- continue
-
- evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)
- self.notifyListeners(evt)
-
- try:
- startIndex = res['content'].index(eventData) - 120
- endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
- self.sf.debug("String not found in content.")
- continue
-
- data = res['content'][startIndex:endIndex]
- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
- self.__name__, evt)
- self.notifyListeners(evt)
- else:
- evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)
- self.notifyListeners(evt)
+
+ data = res['content'][startIndex:endIndex]
+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
+ self.__name__, evt)
+ self.notifyListeners(evt)
+ else:
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)
+ self.notifyListeners(evt)
# End of sfp_onioncity class