SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Update sfp_onionsearchengine
Brendan Colesaabc5f81de76
diff --git a/modules/sfp_onionsearchengine.py b/modules/sfp_onionsearchengine.pyindex bf0a529e..5da504eb 100644--- a/modules/sfp_onionsearchengine.py+++ b/modules/sfp_onionsearchengine.py@@ -11,8 +11,9 @@# Licence: GPL# --------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventimport re+import urllib+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_onionsearchengine(SpiderFootPlugin):"""Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""@@ -20,18 +21,20 @@ class sfp_onionsearchengine(SpiderFootPlugin):# Default optionsopts = {- # We don't bother with pagination as ahmia seems fairly limited in coverage+ 'timeout': 10,+ 'max_pages': 20,'fetchlinks': True,'blacklist': [ '.*://relate.*' ]}# Option descriptionsoptdescs = {+ 'timeout': "Query timeout, in seconds.",+ 'max_pages': "Maximum number of pages of results to fetch.",'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.",'blacklist': "Exclude results from sites matching these patterns."}- # Targetresults = Nonedef setup(self, sfc, userOpts=dict()):@@ -59,42 +62,40 @@ class sfp_onionsearchengine(SpiderFootPlugin):if eventData in self.results:self.sf.debug("Already did a search for " + eventData + ", skipping.")return None- else:- self.results[eventData] = True+ self.results[eventData] = TruekeepGoing = True- paging = ""- pagenr = 1- while keepGoing:+ page = 1+ while keepGoing and page <= int(self.opts['max_pages']):+ # Check if we've been asked to stopif self.checkForStop():return None+ params = {+ 'search': '"' + eventData.encode('raw_unicode_escape') + '"',+ 'submit': 'Search',+ 'page': str(page)+ }+# Sites hosted on the domain- data = self.sf.fetchUrl("https://onionsearchengine.com/search.php?search=\"" + \- eventData.replace(" ", "%20") + "\"&submit=Search" + paging,+ data = self.sf.fetchUrl('https://onionsearchengine.com/search.php?' + urllib.urlencode(params),useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout'])+ timeout=self.opts['timeout'])+if data is None or not data.get('content'):self.sf.info("No results returned from onionsearchengine.com.")return None+ page += 1+if "url.php?u=" not in data['content']:# Work around some kind of bug in the siteif "you didn't submit a keyword" in data['content']:- pagenr += 1- paging = "&page=" + str(pagenr)continuereturn None- # Check if we've been asked to stop- if self.checkForStop():- return None-- if "forward >" in data['content']:- pagenr += 1- paging = "&page=" + str(pagenr)- else:+ if "forward >" not in data['content']:keepGoing = False# Submit the google results for analysis@@ -103,54 +104,60 @@ class sfp_onionsearchengine(SpiderFootPlugin):self.notifyListeners(evt)links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",- data['content'], re.IGNORECASE | re.DOTALL)+ data['content'], re.IGNORECASE | re.DOTALL)for link in links:+ if self.checkForStop():+ return None+if link in self.results:continue- else:- self.results[link] = True- blacklist = False- for r in self.opts['blacklist']:- if re.match(r, link, re.IGNORECASE):- self.sf.debug("Skipping " + link + " as it matches blacklist " + r)- blacklist = True- if blacklist:- continue-- self.sf.debug("Found a darknet mention: " + link)- if self.sf.urlFQDN(link).endswith(".onion"):- if self.checkForStop():- return None- if self.opts['fetchlinks']:- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])-- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- if eventData not in res['content']:- self.sf.debug("Ignoring " + link + " as no mention of " + eventData)- continue- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)- self.notifyListeners(evt)-- try:- startIndex = res['content'].index(eventData) - 120- endIndex = startIndex + len(eventData) + 240- except BaseException as e:- self.sf.debug("String not found in content.")- continue-- data = res['content'][startIndex:endIndex]- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",- self.__name__, evt)- self.notifyListeners(evt)-- else:- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)- self.notifyListeners(evt)+ self.results[link] = True++ blacklist = False+ for r in self.opts['blacklist']:+ if re.match(r, link, re.IGNORECASE):+ self.sf.debug("Skipping " + link + " as it matches blacklist " + r)+ blacklist = True+ if blacklist:+ continue++ self.sf.debug("Found a darknet mention: " + link)++ if not self.sf.urlFQDN(link).endswith(".onion"):+ continue++ if not self.opts['fetchlinks']:+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)+ continue++ res = self.sf.fetchUrl(link,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])++ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")+ continue++ if eventData not in res['content']:+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)+ continue++ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)++ try:+ startIndex = res['content'].index(eventData) - 120+ endIndex = startIndex + len(eventData) + 240+ except BaseException as e:+ self.sf.debug('String "' + eventData + '" not found in content.')+ continue++ data = res['content'][startIndex:endIndex]+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",+ self.__name__, evt)+ self.notifyListeners(evt)# End of sfp_onionsearchengine class