SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Darkweb search module.
Steve Micallefe72d5e4e0b28
diff --git a/evtypes.sql b/evtypes.sqlindex a20f6f82..0624efa5 100644--- a/evtypes.sql+++ b/evtypes.sql@@ -30,6 +30,8 @@ INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES (INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, "DESCRIPTOR");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, "DESCRIPTOR");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, "DATA");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, "DESCRIPTOR");+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, "DATA");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, "ENTITY");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, "ENTITY");INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, "DATA");diff --git a/modules/sfp_darksearch.py b/modules/sfp_darksearch.pynew file mode 100644index 00000000..2a20a218--- /dev/null+++ b/modules/sfp_darksearch.py@@ -0,0 +1,134 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_darksearch+# Purpose: Searches Tor onion site search engines for content related to the+# domain in question.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 15/07/2015+# Copyright: (c) Steve Micallef 2015+# Licence: GPL+# -------------------------------------------------------------------------------++from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+import re++class sfp_darksearch(SpiderFootPlugin):+ """Darknet:Search Tor 'Onion City' search engine for mentions of the target domain."""++ # Default options+ opts = {+ 'pages': 20, # Number of results pages to iterate+ 'fetchlinks': True+ }++ # Option descriptions+ optdescs = {+ 'pages': "Number of results pages to iterate through.",+ 'fetchlinks': "Fetch the darknet pages (via Onion City) to verify they mention your target."+ }++ # Target+ results = list()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = list()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["INTERNET_NAME", "DOMAIN_NAME"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["DARKNET_MENTION_URL", "DARKNET_MENTION_CONTENT",+ "SEARCH_ENGINE_WEB_CONTENT"]++ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if eventData in self.results:+ self.sf.debug("Already did a search for " + eventData + ", skipping.")+ return None+ else:+ self.results.append(eventData)++ # Sites hosted on the domain+ pages = self.sf.googleIterate("site:onion.city+" + eventData,+ dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']))+ if pages is None:+ self.sf.info("No results returned from Google.")+ return None++ for page in pages.keys():+ if page in self.results:+ continue+ else:+ self.results.append(page)++ # Check if we've been asked to stop+ if self.checkForStop():+ return None++ # Submit the google results for analysis+ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],+ self.__name__, event)+ self.notifyListeners(evt)++ links = self.sf.parseLinks(page, pages[page], eventData)+ if len(links) == 0:+ continue++ for link in links:+ if link in self.results:+ continue+ else:+ self.results.append(link)+ self.sf.debug("Found a darknet mention: " + link)+ if self.sf.urlFQDN(link).endswith("onion.city"):+ if self.checkForStop():+ return None++ if self.opts['fetchlinks']:+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])++ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")+ continue++ # Sometimes onion city search results false positives+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ self.sf.debug("Ignoring " + link + " as no mention of " + eventData)+ continue++ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)++ try:+ startIndex = res['content'].index(eventData) - 120+ endIndex = startIndex + len(eventData) + 240+ except BaseException as e:+ self.sf.debug("String not found in content.")+ continue++ data = res['content'][startIndex:endIndex]+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",+ self.__name__, evt)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)+ self.notifyListeners(evt)+++# End of sfp_darksearch classdiff --git a/sfdb.py b/sfdb.pyindex d1835b6d..2d49afb6 100644--- a/sfdb.py+++ b/sfdb.py@@ -97,6 +97,8 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",