SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Added SecurityTrails.com as a data source.
Stevee00de8d6807b
diff --git a/modules/sfp_securitytrails.py b/modules/sfp_securitytrails.pynew file mode 100644index 00000000..b92e70cb--- /dev/null+++ b/modules/sfp_securitytrails.py@@ -0,0 +1,219 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_securitytrails+# Purpose: Query securitytrails using their API+#+# Author: Steve Micallef+#+# Created: 01/02/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import base64+from datetime import datetime+import time+import socket+from netaddr import IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_securitytrails(SpiderFootPlugin):+ """SecurityTrails:Investigate,Passive:Search Engines:apikey:Obtain Passive DNS and other information from SecurityTrails"""++ # Default options+ opts = {+ "api_key": "",+ "verify": True,+ "cohostsamedomain": False+ }++ # Option descriptions+ optdescs = {+ "api_key": "Your SecurityTrails API key",+ "verify": "Verify co-hosts are valid by checking if they still resolve to the shared IP.",+ "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",+ }++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = dict()+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["IP_ADDRESS", "IPV6_ADDRESS", "DOMAIN_NAME",+ "EMAILADDR", "NETBLOCK_OWNER"]++ # What events this module produces+ def producedEvents(self):+ return ["CO_HOSTED_SITE", "AFFILIATE_DOMAIN", "INTERNET_NAME",+ "PROVIDER_HOSTING"]++ # Verify a host resolves to an IP+ def validateIP(self, host, ip):+ try:+ addrs = socket.gethostbyname_ex(host)+ except BaseException as e:+ self.sf.debug("Unable to resolve " + host + ": " + str(e))+ return False++ for addr in addrs:+ if type(addr) == list:+ for a in addr:+ if str(a) == ip:+ return True+ else:+ if str(addr) == ip:+ return True+ return False++ # Search SecurityTrails+ def query(self, qry, querytype, page=1, accum=None):+ info = None++ headers = {+ 'APIKEY': self.opts['api_key']+ }++ if querytype == "domain":+ url = "https://api.securitytrails.com/v1/domain/" + qry + "/subdomains"+ request = None+ else:+ url = "https://api.securitytrails.com/v1/search/list/?page=" + str(page)+ request = '{"filter": { "' + querytype + '": "' + qry + '" } }'+ headers['Content-Type'] = 'application/json'++ res = self.sf.fetchUrl(url , timeout=self.opts['_fetchtimeout'],+ useragent="SpiderFoot", headers=headers,+ postData=request)++ if res['code'] in [ "400", "429", "500", "403" ]:+ self.sf.error("SecurityTrails API key seems to have been rejected or you have exceeded usage limits for the month.", False)+ self.errorState = True+ return None++ if res['content'] is None:+ self.sf.info("No SecurityTrails info found for " + qry)+ return None++ try:+ info = json.loads(res['content'])+ if querytype == "domain":+ return info.get('subdomains', None)+ if info.get("record_count", 0) > 100:+ if len(info.get('records', [])) >= 100:+ # Avoid throttling+ time.sleep(1)+ if accum:+ accum.extend(info.get('records'))+ else:+ accum = info.get('records')+ return self.query(qry, querytype, page+1, accum)+ else:+ # We are at the last page+ accum.extend(info.get('records', []))+ return accum+ else:+ return info.get('records', [])+ except Exception as e:+ self.sf.error("Error processing JSON response from SecurityTrails: " + str(e), False)+ return None++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_securitytrails but did not set an API uid/secret!", False)+ self.errorState = True+ return None++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ if eventName in [ "IP_ADDRESS", "IPV6_ADDRESS", "NETLBLOCK_OWNER"]:+ ip = eventData+ rec = self.query(ip, "ipv4")+ myres = list()+ hosters = list()+ if rec is not None:+ for r in rec:+ if "host_provider" in r:+ for dat in r['host_provider']:+ if dat in hosters:+ continue+ e = SpiderFootEvent("PROVIDER_HOSTING", dat,+ self.__name__, event)+ self.notifyListeners(e)+ hosters.append(dat)+ if "hostname" in r:+ h = r['hostname']+ if not self.opts['cohostsamedomain']:+ if self.getTarget().matches(h, includeParents=True):+ self.sf.debug("Skipping " + h + " because it is on the same domain.")+ continue++ if h not in myres and h != ip:+ if self.opts['verify'] and not self.validateIP(h, ip):+ self.sf.debug("Host no longer resolves to our IP.")+ continue+ myres.append(h.lower())+ e = SpiderFootEvent("CO_HOSTED_SITE", h, self.__name__, event)+ self.notifyListeners(e)++ if eventName in [ "EMAILADDR"]:+ email = eventData+ rec = self.query(email, "whois_email")+ myres = list()+ if rec is not None:+ for r in rec:+ if "hostname" in r:+ if not r['hostname']:+ continue+ h = r['hostname']+ if h not in myres:+ myres.append(h.lower())+ else:+ continue+ e = SpiderFootEvent("AFFILIATE_DOMAIN", h, self.__name__, event)+ self.notifyListeners(e)++ if eventName in [ "DOMAIN_NAME"]:+ domain = eventData+ rec = self.query(domain, "domain")+ myres = list()+ if rec is not None:+ for h in rec:+ if h not in myres:+ myres.append(h.lower())+ else:+ continue+ e = SpiderFootEvent("INTERNET_NAME", h + "." + domain,+ self.__name__, event)+ self.notifyListeners(e)+++# End of sfp_securitytrails class