SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
feat: update module onioncity
Amr Awad5fbcaf2ed0f0
diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.pyindex 7d33b1ea..83ad7146 100644--- a/modules/sfp_googlesearch.py+++ b/modules/sfp_googlesearch.py@@ -72,7 +72,7 @@ class sfp_googlesearch(SpiderFootPlugin):},)if results is None:- # Failed to talk to the bing API or no results returned+ # Failed to talk to the Google API or no results returnedreturn Noneurls = results["urls"]@@ -91,10 +91,10 @@ class sfp_googlesearch(SpiderFootPlugin):self.notifyListeners(evt)if internal_links:- # Submit the bing results for analysis- bingsearch_url = results["webSearchUrl"]+ # Submit the Google results for analysis+ googlesearch_url = results["webSearchUrl"]response = self.sf.fetchUrl(- bingsearch_url,+ googlesearch_url,timeout=self.opts["_fetchtimeout"],useragent=self.opts["_useragent"],)@@ -104,6 +104,6 @@ class sfp_googlesearch(SpiderFootPlugin):)self.notifyListeners(evt)else:- self.sf.error("Failed to fetch bing web search URL", exception=False)+ self.sf.error("Failed to fetch Google web search URL", exception=False)# End of sfp_googlesearch classdiff --git a/modules/sfp_googlesearchdomain.py b/modules/sfp_googlesearchdomain.pyindex 0216009e..7adf0521 100644--- a/modules/sfp_googlesearchdomain.py+++ b/modules/sfp_googlesearchdomain.py@@ -85,7 +85,7 @@ class sfp_googlesearchdomain(SpiderFootPlugin):},)if results is None:- # Failed to talk to the bing API or no results returned+ # Failed to talk to the Google API or no results returnedreturn Noneurls = results["urls"]@@ -104,10 +104,10 @@ class sfp_googlesearchdomain(SpiderFootPlugin):self.notifyListeners(evt)if internal_links:- # Submit the bing results for analysis- bingsearch_url = results["webSearchUrl"]+ # Submit the Google results for analysis+ googlesearch_url = results["webSearchUrl"]response = self.sf.fetchUrl(- bingsearch_url,+ googlesearch_url,timeout=self.opts["_fetchtimeout"],useragent=self.opts["_useragent"],)@@ -117,6 +117,6 @@ class sfp_googlesearchdomain(SpiderFootPlugin):)self.notifyListeners(evt)else:- self.sf.error("Failed to fetch bing web search URL", exception=False)+ self.sf.error("Failed to fetch Google web search URL", exception=False)# End of sfp_googlesearchdomain classdiff --git a/modules/sfp_onioncity.py b/modules/sfp_onioncity.pyindex e1e6f38b..0837f9b6 100644--- a/modules/sfp_onioncity.py+++ b/modules/sfp_onioncity.py@@ -15,18 +15,18 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventimport reclass sfp_onioncity(SpiderFootPlugin):- """Onion.link:Footprint,Investigate:Search Engines:errorprone:Search Tor 'Onion City' search engine for mentions of the target domain."""+ """Onion.link:Footprint,Investigate:Search Engines:apikey:Search Tor 'Onion City' search engine for mentions of the target domain."""# Default optionsopts = {- 'pages': 20, # Number of results pages to iterate- 'fetchlinks': True+ "api_key": "", "cse_id": "", 'fetchlinks': True}# Option descriptionsoptdescs = {- 'pages': "Number of results pages to iterate through.",+ "api_key": "Google API Key.",+ "cse_id": "Google Custom Search Engine ID.",'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target."}@@ -63,73 +63,82 @@ class sfp_onioncity(SpiderFootPlugin):self.results.append(eventData)# Sites hosted on the domain- pages = self.sf.googleIterate("site:onion.link+" + eventData,- dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))- if pages is None:- self.sf.info("No results returned from Google.")+ results = self.sf.googleIterate(+ searchString="site:onion.link+" + eventData,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],+ "api_key": self.opts["api_key"],+ "cse_id": self.opts["cse_id"],+ },+ )+ if results is None:+ # Failed to talk to the bing API or no results returnedreturn None- for page in pages.keys():- if page in self.results:- continue- else:- self.results.append(page)+ urls = results["urls"]+ new_links = list(set(urls) - set(self.results))++ # Add new links to results+ self.results.extend(new_links)++ # Submit the Google results for analysis+ googlesearch_url = results["webSearchUrl"]+ response = self.sf.fetchUrl(+ googlesearch_url,+ timeout=self.opts["_fetchtimeout"],+ useragent=self.opts["_useragent"],+ )+ if response['status'] == 'OK':+ evt = SpiderFootEvent(+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event+ )+ self.notifyListeners(evt)+ else:+ self.sf.error("Failed to fetch Google web search URL", exception=False)- # Check if we've been asked to stop- if self.checkForStop():- return None+ # Check if we've been asked to stop+ if self.checkForStop():+ return None- # Submit the google results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],- self.__name__, event)- self.notifyListeners(evt)+ darknet_links = [+ link for link in new_links if self.sf.urlFQDN(link).endswith(".onion.link")+ ]++ for link in darknet_links:+ self.sf.debug("Found a darknet mention: " + link)+ torlink = link.replace(".onion.link", ".onion")+ if self.opts['fetchlinks']:+ res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])- links = self.sf.parseLinks(page, pages[page], eventData)- if len(links) == 0:- continue+ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")+ continue++ # Sometimes onion city search results false positives+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)+ continue- for link in links:- if link in self.results:+ evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)+ self.notifyListeners(evt)++ try:+ startIndex = res['content'].index(eventData) - 120+ endIndex = startIndex + len(eventData) + 240+ except BaseException as e:+ self.sf.debug("String not found in content.")continue- else:- self.results.append(link)- self.sf.debug("Found a darknet mention: " + link)- if self.sf.urlFQDN(link).endswith(".onion.link"):- if self.checkForStop():- return None- torlink = link.replace(".onion.link", ".onion")- if self.opts['fetchlinks']:- res = self.sf.fetchUrl(torlink, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])-- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- # Sometimes onion city search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:- self.sf.debug("Ignoring " + link + " as no mention of " + eventData)- continue-- evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)- self.notifyListeners(evt)-- try:- startIndex = res['content'].index(eventData) - 120- endIndex = startIndex + len(eventData) + 240- except BaseException as e:- self.sf.debug("String not found in content.")- continue-- data = res['content'][startIndex:endIndex]- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",- self.__name__, evt)- self.notifyListeners(evt)- else:- evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)- self.notifyListeners(evt)++ data = res['content'][startIndex:endIndex]+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",+ self.__name__, evt)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent("DARKNET_MENTION_URL", torlink, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_onioncity class