Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Update sfp_onionsearchengine

Brendan Colesaabc5f81de76
diff --git a/modules/sfp_onionsearchengine.py b/modules/sfp_onionsearchengine.py
index bf0a529e..5da504eb 100644
--- a/modules/sfp_onionsearchengine.py
+++ b/modules/sfp_onionsearchengine.py
@@ -11,8 +11,9 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
import re
+import urllib
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_onionsearchengine(SpiderFootPlugin):
"""Onionsearchengine.com:Footprint,Investigate:Search Engines::Search Tor onionsearchengine.com for mentions of the target domain."""
@@ -20,18 +21,20 @@ class sfp_onionsearchengine(SpiderFootPlugin):
# Default options
opts = {
- # We don't bother with pagination as ahmia seems fairly limited in coverage
+ 'timeout': 10,
+ 'max_pages': 20,
'fetchlinks': True,
'blacklist': [ '.*://relate.*' ]
}
# Option descriptions
optdescs = {
+ 'timeout': "Query timeout, in seconds.",
+ 'max_pages': "Maximum number of pages of results to fetch.",
'fetchlinks': "Fetch the darknet pages (via TOR, if enabled) to verify they mention your target.",
'blacklist': "Exclude results from sites matching these patterns."
}
- # Target
results = None
def setup(self, sfc, userOpts=dict()):
@@ -59,42 +62,40 @@ class sfp_onionsearchengine(SpiderFootPlugin):
if eventData in self.results:
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
- else:
- self.results[eventData] = True
+ self.results[eventData] = True
keepGoing = True
- paging = ""
- pagenr = 1
- while keepGoing:
+ page = 1
+ while keepGoing and page <= int(self.opts['max_pages']):
+ # Check if we've been asked to stop
if self.checkForStop():
return None
+ params = {
+ 'search': '"' + eventData.encode('raw_unicode_escape') + '"',
+ 'submit': 'Search',
+ 'page': str(page)
+ }
+
# Sites hosted on the domain
- data = self.sf.fetchUrl("https://onionsearchengine.com/search.php?search=\"" + \
- eventData.replace(" ", "%20") + "\"&submit=Search" + paging,
+ data = self.sf.fetchUrl('https://onionsearchengine.com/search.php?' + urllib.urlencode(params),
useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout'])
+ timeout=self.opts['timeout'])
+
if data is None or not data.get('content'):
self.sf.info("No results returned from onionsearchengine.com.")
return None
+ page += 1
+
if "url.php?u=" not in data['content']:
# Work around some kind of bug in the site
if "you didn't submit a keyword" in data['content']:
- pagenr += 1
- paging = "&page=" + str(pagenr)
continue
return None
- # Check if we've been asked to stop
- if self.checkForStop():
- return None
-
- if "forward >" in data['content']:
- pagenr += 1
- paging = "&page=" + str(pagenr)
- else:
+ if "forward >" not in data['content']:
keepGoing = False
# Submit the google results for analysis
@@ -103,54 +104,60 @@ class sfp_onionsearchengine(SpiderFootPlugin):
self.notifyListeners(evt)
links = re.findall("url\.php\?u=(.[^\"\']+)[\"\']",
- data['content'], re.IGNORECASE | re.DOTALL)
+ data['content'], re.IGNORECASE | re.DOTALL)
for link in links:
+ if self.checkForStop():
+ return None
+
if link in self.results:
continue
- else:
- self.results[link] = True
- blacklist = False
- for r in self.opts['blacklist']:
- if re.match(r, link, re.IGNORECASE):
- self.sf.debug("Skipping " + link + " as it matches blacklist " + r)
- blacklist = True
- if blacklist:
- continue
-
- self.sf.debug("Found a darknet mention: " + link)
- if self.sf.urlFQDN(link).endswith(".onion"):
- if self.checkForStop():
- return None
- if self.opts['fetchlinks']:
- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
-
- if res['content'] is None:
- self.sf.debug("Ignoring " + link + " as no data returned")
- continue
-
- if eventData not in res['content']:
- self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
- continue
- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
- self.notifyListeners(evt)
-
- try:
- startIndex = res['content'].index(eventData) - 120
- endIndex = startIndex + len(eventData) + 240
- except BaseException as e:
- self.sf.debug("String not found in content.")
- continue
-
- data = res['content'][startIndex:endIndex]
- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
- self.__name__, evt)
- self.notifyListeners(evt)
-
- else:
- evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
- self.notifyListeners(evt)
+ self.results[link] = True
+
+ blacklist = False
+ for r in self.opts['blacklist']:
+ if re.match(r, link, re.IGNORECASE):
+ self.sf.debug("Skipping " + link + " as it matches blacklist " + r)
+ blacklist = True
+ if blacklist:
+ continue
+
+ self.sf.debug("Found a darknet mention: " + link)
+
+ if not self.sf.urlFQDN(link).endswith(".onion"):
+ continue
+
+ if not self.opts['fetchlinks']:
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
+ self.notifyListeners(evt)
+ continue
+
+ res = self.sf.fetchUrl(link,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
+
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
+ continue
+
+ if eventData not in res['content']:
+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
+ continue
+
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
+ self.notifyListeners(evt)
+
+ try:
+ startIndex = res['content'].index(eventData) - 120
+ endIndex = startIndex + len(eventData) + 240
+ except BaseException as e:
+ self.sf.debug('String "' + eventData + '" not found in content.')
+ continue
+
+ data = res['content'][startIndex:endIndex]
+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
+ self.__name__, evt)
+ self.notifyListeners(evt)
# End of sfp_onionsearchengine class