SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
New version, logo and CLI functionality.
Steve Micallef745557633f27
diff --git a/Dockerfile b/Dockerfileindex f3a40b36..ac5fb314 100644--- a/Dockerfile+++ b/Dockerfile@@ -14,8 +14,8 @@# sudo docker run -it -p 5001:5001 spiderfoot# Pull the base image.-FROM alpine:3.7-ENV SPIDERFOOT_VERSION 2.12.0+FROM alpine:latest+ENV SPIDERFOOT_VERSION 3.0.0COPY requirements.txt .# Run everything as one command so that only one layer is createddiff --git a/dyn/HEADER.tmpl b/dyn/HEADER.tmplindex 7ff817dd..fbe91d40 100644--- a/dyn/HEADER.tmpl+++ b/dyn/HEADER.tmpl@@ -4,7 +4,7 @@<script type='text/javascript' src='${docroot}/static/js/viz.js'></script><head><meta charset="utf-8">- <title>SpiderFoot v2.12</title>+ <title>SpiderFoot v3.0</title><meta name="viewport" content="width=device-width, initial-scale=1.0"><meta name="description" content=""><meta name="author" content="">@@ -15,23 +15,21 @@<link href="${docroot}/static/css/spiderfoot.css" rel="stylesheet"><style>body {- padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */}- .brand-logo- {- display: block;- float: left;- padding: 10px 20px 10px;- margin-left: -20px;- font-size: 20px;- font-weight: 200;- color: #777777;- text-shadow: 0 1px 0 #ffffff;- background: url(static/img/spiderfoot.png) no-repeat center center;- background-size: 38px 38px;- height: 20px;- width: 20px;- }+ .navbar-brand {+ image-rendering: -o-crisp-edges; /* Opera */+ image-rendering: -webkit-optimize-contrast;/* Webkit (non-standard naming) */+ image-rendering: crisp-edges;+ -ms-interpolation-mode: nearest-neighbor; /* IE (non-standard property) */+ }+ .brand-logo {+ display: block;+ float: left;+ padding-top: 5px;+ padding-right: 20px;+ }+</style><link href="${docroot}/static/css/bootstrap-responsive.css" rel="stylesheet">@@ -52,8 +50,7 @@<span class="icon-bar"></span><span class="icon-bar"></span></a>- <a class="brand-logo" href="${docroot}/"></a>- <a class="brand" href="${docroot}/">SpiderFoot</a>+ <a class="navbar-brand brand-logo" href="${docroot}/"><img style='height: 30px' src="${docroot}/static/img/spiderfoot-header.png" /></a><div class="nav-collapse collapse"><ul class="nav"><li@@ -82,13 +79,13 @@<div id="aboutmodal" class="modal hide fade"><div class="modal-header">- <h3>SpiderFoot 2.12</h3>+ <h3>SpiderFoot 3.0</h3></div><div class="modal-body"><p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='http://www.binarypool.com' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p><p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:[email protected]>[email protected]</a>.<br>- <p>You are running version <b>2.12</b>.</p>+ <p>You are running version <b>3.0</b>.</p><p> Visit the project website at <a href='http://www.spiderfoot.net' target=_new>http://www.spiderfoot.net</a>.</p><p> Check out the Github repo at <a href='https://github.com/smicallef/spiderfoot' target=_new>https://github.com/smicallef/spiderfoot</a>.</p><p> Latest updates @ Twitter <a href='https://twitter.com/binarypool' target=_new>https://twitter.com/binarypool</a>.</p>diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pynew file mode 100644index 00000000..99b8dd9a--- /dev/null+++ b/modules/sfp__stor_stdout.py@@ -0,0 +1,84 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_stor_stdout+# Purpose: SpiderFoot plug-in for dumping events to standard output.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 22/10/2018+# Copyright: (c) Steve Micallef 2018+# Licence: GPL+# -------------------------------------------------------------------------------++from sflib import SpiderFoot, SpiderFootPlugin+import json+++class sfp__stor_stdout(SpiderFootPlugin):+ """Command-line output::::Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line."""++ # Default options+ opts = {+ "_format": "tab", # tab, csv, json+ "_requested": [],+ "_showonlyrequested": False,+ "_stripnewline": False,+ "_maxlength": 0,+ "_eventtypes": dict()+ }++ # Option descriptions+ optdescs = {+ }++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ # Because this is a storage plugin, we are interested in everything so we+ # can store all events for later analysis.+ def watchedEvents(self):+ return ["*"]++ def output(self, event):+ if type(event.data) in [list, dict]:+ data = unicode(str(event.data), 'utf-8', errors='replace')+ else:+ data = event.data++ if type(data) != unicode:+ data = unicode(event.data, 'utf-8', errors='replace')++ if self.opts['_stripnewline']:+ data = data.replace("\n", "").replace("\r", "")++ if self.opts['_maxlength'] > 0:+ data = data[0:self.opts['_maxlength']]++ if self.opts['_format'] == "tab":+ print u'{0:45} {1}'.format(self.opts['_eventtypes'][event.eventType], data)++ if self.opts['_format'] == "csv":+ print self.opts['_eventtypes'][event.eventType] + "," + data++ if self.opts['_format'] == "json":+ d = event.asDict()+ d['type'] = self.opts['_eventtypes'][event.eventType]+ print json.dumps(d)+++ # Handle events sent to this module+ def handleEvent(self, sfEvent):+ if sfEvent.eventType == "ROOT":+ return None++ if self.opts['_showonlyrequested']:+ if sfEvent.eventType in self.opts['_requested']:+ self.output(sfEvent)+ else:+ self.output(sfEvent)++# End of sfp__stor_stdout classdiff --git a/modules/sfp_tool_dnstwist.py b/modules/sfp_tool_dnstwist.pyindex c1c2a2d4..a66e9bff 100644--- a/modules/sfp_tool_dnstwist.py+++ b/modules/sfp_tool_dnstwist.py@@ -23,12 +23,13 @@ class sfp_tool_dnstwist(SpiderFootPlugin):# Default optionsopts = {- # Commonly used ports on external-facing systems+ 'pythonpath': "python"'dnstwistpath': "",}# Option descriptionsoptdescs = {+ 'pythonpath': "Path to Python interpreter to use for DNSTwist. If just 'python' then it should be in your PATH.",'dnstwistpath': "Path to the where the dnstwist.py file lives. Must be set.",}@@ -89,7 +90,7 @@ class sfp_tool_dnstwist(SpiderFootPlugin):return Nonetry:- p = Popen(["python", exe, "-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)+ p = Popen([self.opts['pythonpath'], exe, "-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)stdout, stderr = p.communicate(input=None)if p.returncode == 0:content = stdoutdiff --git a/sf.py b/sf.pyindex 2a68771d..f1736d20 100755--- a/sf.py+++ b/sf.py@@ -14,6 +14,10 @@import sysimport osimport inspect+import signal+import time+import argparse+from copy import deepcopy# Look under ext ford 3rd party dependenciescmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile(inspect.currentframe()))[0], "ext")))@@ -52,20 +56,24 @@ import cherrypyimport randomfrom cherrypy.lib import auth_digestfrom sflib import SpiderFoot+from sfdb import SpiderFootDbfrom sfwebui import SpiderFootWebUi+from sfscan import SpiderFootScanner# 'Global' configuration options# These can be overriden on a per-module basis, and some will# be overridden from saved configuration settings stored in the DB.sfConfig = {'_debug': False, # Debug+ '__logging': True, # Logging in general'__blocknotif': False, # Block notifications+ '_fatalerrors': False,'_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests'_dnsserver': '', # Override the default resolver'_fetchtimeout': 5, # number of seconds before giving up on a fetch'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat','_internettlds_cache': 72,- '__version__': '2.12',+ '__version__': '3.0','__database': 'spiderfoot.db','__webaddr': '127.0.0.1','__webport': 5001,@@ -77,7 +85,8 @@ sfConfig = {'_socks4user': '','_socks5pwd': '','_socks6dns': True,- '_torctlport': 9051+ '_torctlport': 9051,+ '__logstdout': False}sfOptdescs = {@@ -94,20 +103,59 @@ sfOptdescs = {'_socks5pwd': "SOCKS Password. Valid only for SOCKS5 servers.",'_socks6dns': "Resolve DNS through the SOCKS proxy? Has no affect when TOR is used: Will always be True.",'_torctlport': "The port TOR is taking control commands on. This is necessary for SpiderFoot to tell TOR to re-circuit when it suspects anonymity is compromised.",+ '_fatalerrors': "Abort the scan when modules encounter exceptions.",'_modulesenabled': "Modules enabled for the scan." # This is a hack to get a description for an option not actually available.}+scanId = None+dbh = None++def handle_abort(signal, frame):+ print "[*] Aborting..."+ if scanId and dbh:+ dbh.scanInstanceSet(scanId, None, None, "ABORTED")+ sys.exit(-1)+if __name__ == '__main__':- if len(sys.argv) > 1:- (addr, port) = sys.argv[1].split(":")- sfConfig['__webaddr'] = addr- sfConfig['__webport'] = int(port)+ # Legacy way to run the server+ args = None+ if (len(sys.argv) > 1 and ":" in sys.argv[1]) or len(sys.argv) == 1:+ if len(sys.argv) > 1:+ (addr, port) = sys.argv[1].split(":")+ sfConfig['__webaddr'] = addr+ sfConfig['__webport'] = int(port)+ sfConfig['__logstdout'] = False+ else:+ p = argparse.ArgumentParser(description='SpiderFoot 3.0: Open Source Intelligence Automation.')+ p.add_argument("-d", "--debug", action='store_true', help="Enable debug output.")+ p.add_argument("-m", metavar="mod1,mod2,...", type=str, help="Modules to enable.")+ p.add_argument("-M", "--modules", action='store_true', help="List available modules.")+ p.add_argument("-s", metavar="TARGET", help="Target for the scan.")+ p.add_argument("-t", metavar="type1,type2,...", type=str, help="Event types to enable.")+ p.add_argument("-T", "--types", action='store_true', help="List available event types.")+ p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default.")+ p.add_argument("-n", action='store_true', help="Strip newlines from data.")+ p.add_argument("-S", metavar="LENGTH", type=int, help="Maximum data length to display. By default, all data is shown.")+ p.add_argument("-f", action='store_true', help="Filter out other event types that weren't requested with -t.")+ p.add_argument("-F", metavar="FILTER", type=str, help="Filter out a set of event types.")+ p.add_argument("-q", action='store_true', help="Disable logging.")+ args = p.parse_args()- sf = SpiderFoot(sfConfig)- sfModules = dict()+ sfConfig['__logstdout'] = True++ if args.debug:+ sfConfig['_debug'] = True+ else:+ sfConfig['_debug'] = False++ if args.q:+ sfConfig['__logging'] = False++ sfModules = dict()+ sft = SpiderFoot(sfConfig)# Go through each module in the modules directory with a .py extension- for filename in os.listdir(sf.myPath() + '/modules/'):+ for filename in os.listdir(sft.myPath() + '/modules/'):if filename.startswith("sfp_") and filename.endswith(".py"):# Skip the module template and debugging modulesif filename == "sfp_template.py" or filename == 'sfp_stor_print.py':@@ -139,6 +187,124 @@ if __name__ == '__main__':# Add descriptions of the global config optionssfConfig['__globaloptdescs__'] = sfOptdescs+ sf = SpiderFoot(sfConfig)+ dbh = SpiderFootDb(sfConfig)++ if args:+ if args.modules:+ print "Modules available:"+ for m in sorted(sfModules.keys()):+ if "__" in m:+ continue+ print '{0:25} {1}'.format(m, sfModules[m]['descr'])+ sys.exit(0)++ if args.types:+ print "Types available:"+ typedata = dbh.eventTypes()+ types = dict()+ for r in typedata:+ types[r[1]] = r[0]++ for t in sorted(types.keys()):+ print '{0:45} {1}'.format(t, types[t])+ sys.exit(0)++ if not args.s:+ print "You must specify a target when running in scan mode. Try sf.py --help for guidance."+ sys.exit(-1)++ target = args.s+ targetType = sf.targetType(args.s)++ modlist = list()+ if not args.t and not args.m:+ print "WARNING: You didn't specify any modules or types, so all will be enabled."+ for m in sfModules.keys():+ if "__" in m:+ continue+ modlist.append(m)++ signal.signal(signal.SIGINT, handle_abort)+ # If the user is scanning by type..+ # 1. Find modules producing that type+ if args.t:+ types = args.t+ modlist = sf.modulesProducing(types)+ newmods = deepcopy(modlist)+ newmodcpy = deepcopy(newmods)+ # 2. For each type those modules consume, get modules producing+ while len(newmodcpy) > 0:+ for etype in sf.eventsToModules(newmodcpy):+ xmods = sf.modulesProducing([etype])+ for mod in xmods:+ if mod not in modlist:+ modlist.append(mod)+ newmods.append(mod)+ newmodcpy = deepcopy(newmods)+ newmods = list()++ # Easier if scanning by module+ if args.m:+ modlist = args.m.split(",")++ # Add sfp__stor_stdout to the module list+ outputformat = "tab"+ typedata = dbh.eventTypes()+ types = dict()+ for r in typedata:+ types[r[1]] = r[0]++ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_eventtypes'] = types+ if args.f:+ if args.f and not args.t:+ print "You can only use -f with -t. Use --help for guidance."+ sys.exit(-1)+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_showonlyrequested'] = True+ if args.F:+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_requested'] = args.F.split(",")+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_showonlyrequested'] = True+ if args.o:+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_format'] = args.o+ if args.t:+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_requested'] = args.t.split(",")+ if args.n:+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_stripnewline'] = True+ if args.S:+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_maxlength'] = args.S++ modlist += ["sfp__stor_db", "sfp__stor_stdout"]++ # Run the scan+ if not args.q:+ print "[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)+ cfg = sf.configUnserialize(dbh.configGet(), sfConfig)+ scanId = sf.genScanInstanceGUID(target)++ # Debug mode is a variable that gets stored to the DB, so re-apply it+ if args.debug:+ cfg['_debug'] = True+ else:+ cfg['_debug'] = False++ t = SpiderFootScanner(target, target, targetType, scanId,+ modlist, cfg, dict())+ t.daemon = True+ t.start()++ while True:+ info = dbh.scanInstanceGet(scanId)+ if not info:+ time.sleep(1)+ continue+ if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:+ if not args.q:+ print "[*] Scan completed with status " + info[5]+ sys.exit(0)+ time.sleep(1)++ sys.exit(0)+# Start the web server so you can start looking at resultsprint "Starting web server at http://" + sfConfig['__webaddr'] + \":" + str(sfConfig['__webport']) + sfConfig['__docroot'] + " ..."diff --git a/sfcli.py b/sfcli.pyindex 3f3a60f3..53d2dff3 100644--- a/sfcli.py+++ b/sfcli.py@@ -35,7 +35,7 @@ class bcolors:GREYBLUE_DARK = '\x1b[38;5;24m'class SpiderFootCli(cmd.Cmd):- version = "2.12"+ version = "3.0"pipecmd = Noneoutput = Nonemodules = []diff --git a/static/img/spiderfoot-icon.png b/static/img/spiderfoot-icon.pngindex 11cc6b5c..706bc826 100644Binary files a/static/img/spiderfoot-icon.png and b/static/img/spiderfoot-icon.png differ