Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Darkweb search module.

Steve Micallefe72d5e4e0b28
diff --git a/evtypes.sql b/evtypes.sql
index a20f6f82..0624efa5 100644
--- a/evtypes.sql
+++ b/evtypes.sql
@@ -30,6 +30,8 @@ INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES (
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, "DESCRIPTOR");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, "DESCRIPTOR");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, "DATA");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, "DATA");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, "ENTITY");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, "ENTITY");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, "DATA");
diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.py
new file mode 100644
index 00000000..2a20a218
--- /dev/null
+++ b/modules/sfp_darksearch.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_darksearch
+# Purpose: Searches Tor onion site search engines for content related to the
+# domain in question.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 15/07/2015
+# Copyright: (c) Steve Micallef 2015
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+import re
+
+class sfp_darksearch(SpiderFootPlugin):
+ """Darknet:Search Tor 'Onion City' search engine for mentions of the target domain."""
+
+ # Default options
+ opts = {
+ 'pages': 20, # Number of results pages to iterate
+ 'fetchlinks': True
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'pages': "Number of results pages to iterate through.",
+ 'fetchlinks': "Fetch the darknet pages (via Onion City) to verify they mention your target."
+ }
+
+ # Target
+ results = list()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = list()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ["INTERNET_NAME", "DOMAIN_NAME"]
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT",
+ "SEARCH_ENGINE_WEB_CONTENT"]
+
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ if eventData in self.results:
+ self.sf.debug("Already did a search for " + eventData + ", skipping.")
+ return None
+ else:
+ self.results.append(eventData)
+
+ # Sites hosted on the domain
+ pages = self.sf.googleIterate("site:onion.city+" + eventData,
+ dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']))
+ if pages is None:
+ self.sf.info("No results returned from Google.")
+ return None
+
+ for page in pages.keys():
+ if page in self.results:
+ continue
+ else:
+ self.results.append(page)
+
+ # Check if we've been asked to stop
+ if self.checkForStop():
+ return None
+
+ # Submit the google results for analysis
+ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],
+ self.__name__, event)
+ self.notifyListeners(evt)
+
+ links = self.sf.parseLinks(page, pages[page], eventData)
+ if len(links) == 0:
+ continue
+
+ for link in links:
+ if link in self.results:
+ continue
+ else:
+ self.results.append(link)
+ self.sf.debug("Found a darknet mention: " + link)
+ if self.sf.urlFQDN(link).endswith("onion.city"):
+ if self.checkForStop():
+ return None
+
+ if self.opts['fetchlinks']:
+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
+
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
+ continue
+
+ # Sometimes onion city search results false positives
+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)
+ continue
+
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
+ self.notifyListeners(evt)
+
+ try:
+ startIndex = res['content'].index(eventData) - 120
+ endIndex = startIndex + len(eventData) + 240
+ except BaseException as e:
+ self.sf.debug("String not found in content.")
+ continue
+
+ data = res['content'][startIndex:endIndex]
+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
+ self.__name__, evt)
+ self.notifyListeners(evt)
+ else:
+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
+ self.notifyListeners(evt)
+
+
+# End of sfp_darksearch class
diff --git a/sfdb.py b/sfdb.py
index d1835b6d..2d49afb6 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -97,6 +97,8 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",