Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

New version, logo and CLI functionality.

Steve Micallef745557633f27
diff --git a/Dockerfile b/Dockerfile
index f3a40b36..ac5fb314 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,8 +14,8 @@
# sudo docker run -it -p 5001:5001 spiderfoot
# Pull the base image.
-FROM alpine:3.7
-ENV SPIDERFOOT_VERSION 2.12.0
+FROM alpine:latest
+ENV SPIDERFOOT_VERSION 3.0.0
COPY requirements.txt .
# Run everything as one command so that only one layer is created
diff --git a/dyn/HEADER.tmpl b/dyn/HEADER.tmpl
index 7ff817dd..fbe91d40 100644
--- a/dyn/HEADER.tmpl
+++ b/dyn/HEADER.tmpl
@@ -4,7 +4,7 @@
<script type='text/javascript' src='${docroot}/static/js/viz.js'></script>
<head>
<meta charset="utf-8">
- <title>SpiderFoot v2.12</title>
+ <title>SpiderFoot v3.0</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta name="description" content="">
<meta name="author" content="">
@@ -15,23 +15,21 @@
<link href="${docroot}/static/css/spiderfoot.css" rel="stylesheet">
<style>
body {
- padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
+ padding-top: 60px; /* 60px to make the container go all the way to the bottom of the topbar */
}
- .brand-logo
- {
- display: block;
- float: left;
- padding: 10px 20px 10px;
- margin-left: -20px;
- font-size: 20px;
- font-weight: 200;
- color: #777777;
- text-shadow: 0 1px 0 #ffffff;
- background: url(static/img/spiderfoot.png) no-repeat center center;
- background-size: 38px 38px;
- height: 20px;
- width: 20px;
- }
+ .navbar-brand {
+ image-rendering: -o-crisp-edges; /* Opera */
+ image-rendering: -webkit-optimize-contrast;/* Webkit (non-standard naming) */
+ image-rendering: crisp-edges;
+ -ms-interpolation-mode: nearest-neighbor; /* IE (non-standard property) */
+ }
+ .brand-logo {
+ display: block;
+ float: left;
+ padding-top: 5px;
+ padding-right: 20px;
+ }
+
</style>
<link href="${docroot}/static/css/bootstrap-responsive.css" rel="stylesheet">
@@ -52,8 +50,7 @@
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</a>
- <a class="brand-logo" href="${docroot}/"></a>
- <a class="brand" href="${docroot}/">SpiderFoot</a>
+ <a class="navbar-brand brand-logo" href="${docroot}/"><img style='height: 30px' src="${docroot}/static/img/spiderfoot-header.png" /></a>
<div class="nav-collapse collapse">
<ul class="nav">
<li
@@ -82,13 +79,13 @@
<div id="aboutmodal" class="modal hide fade">
<div class="modal-header">
- <h3>SpiderFoot 2.12</h3>
+ <h3>SpiderFoot 3.0</h3>
</div>
<div class="modal-body">
<p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='http://www.binarypool.com' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>
<p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:[email protected]>[email protected]</a>.
<br>
- <p>You are running version <b>2.12</b>.</p>
+ <p>You are running version <b>3.0</b>.</p>
<p> Visit the project website at <a href='http://www.spiderfoot.net' target=_new>http://www.spiderfoot.net</a>.</p>
<p> Check out the Github repo at <a href='https://github.com/smicallef/spiderfoot' target=_new>https://github.com/smicallef/spiderfoot</a>.</p>
<p> Latest updates @ Twitter <a href='https://twitter.com/binarypool' target=_new>https://twitter.com/binarypool</a>.</p>
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
new file mode 100644
index 00000000..99b8dd9a
--- /dev/null
+++ b/modules/sfp__stor_stdout.py
@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_stor_stdout
+# Purpose: SpiderFoot plug-in for dumping events to standard output.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 22/10/2018
+# Copyright: (c) Steve Micallef 2018
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+from sflib import SpiderFoot, SpiderFootPlugin
+import json
+
+
+class sfp__stor_stdout(SpiderFootPlugin):
+ """Command-line output::::Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line."""
+
+ # Default options
+ opts = {
+ "_format": "tab", # tab, csv, json
+ "_requested": [],
+ "_showonlyrequested": False,
+ "_stripnewline": False,
+ "_maxlength": 0,
+ "_eventtypes": dict()
+ }
+
+ # Option descriptions
+ optdescs = {
+ }
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ # Because this is a storage plugin, we are interested in everything so we
+ # can store all events for later analysis.
+ def watchedEvents(self):
+ return ["*"]
+
+ def output(self, event):
+ if type(event.data) in [list, dict]:
+ data = unicode(str(event.data), 'utf-8', errors='replace')
+ else:
+ data = event.data
+
+ if type(data) != unicode:
+ data = unicode(event.data, 'utf-8', errors='replace')
+
+ if self.opts['_stripnewline']:
+ data = data.replace("\n", "").replace("\r", "")
+
+ if self.opts['_maxlength'] > 0:
+ data = data[0:self.opts['_maxlength']]
+
+ if self.opts['_format'] == "tab":
+ print u'{0:45} {1}'.format(self.opts['_eventtypes'][event.eventType], data)
+
+ if self.opts['_format'] == "csv":
+ print self.opts['_eventtypes'][event.eventType] + "," + data
+
+ if self.opts['_format'] == "json":
+ d = event.asDict()
+ d['type'] = self.opts['_eventtypes'][event.eventType]
+ print json.dumps(d)
+
+
+ # Handle events sent to this module
+ def handleEvent(self, sfEvent):
+ if sfEvent.eventType == "ROOT":
+ return None
+
+ if self.opts['_showonlyrequested']:
+ if sfEvent.eventType in self.opts['_requested']:
+ self.output(sfEvent)
+ else:
+ self.output(sfEvent)
+
+# End of sfp__stor_stdout class
diff --git a/modules/sfp_tool_dnstwist.py b/modules/sfp_tool_dnstwist.py
index c1c2a2d4..a66e9bff 100644
--- a/modules/sfp_tool_dnstwist.py
+++ b/modules/sfp_tool_dnstwist.py
@@ -23,12 +23,13 @@ class sfp_tool_dnstwist(SpiderFootPlugin):
# Default options
opts = {
- # Commonly used ports on external-facing systems
+ 'pythonpath': "python"
'dnstwistpath': "",
}
# Option descriptions
optdescs = {
+ 'pythonpath': "Path to Python interpreter to use for DNSTwist. If just 'python' then it should be in your PATH.",
'dnstwistpath': "Path to the where the dnstwist.py file lives. Must be set.",
}
@@ -89,7 +90,7 @@ class sfp_tool_dnstwist(SpiderFootPlugin):
return None
try:
- p = Popen(["python", exe, "-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)
+ p = Popen([self.opts['pythonpath'], exe, "-f", "json", "-r", eventData], stdout=PIPE, stderr=PIPE)
stdout, stderr = p.communicate(input=None)
if p.returncode == 0:
content = stdout
diff --git a/sf.py b/sf.py
index 2a68771d..f1736d20 100755
--- a/sf.py
+++ b/sf.py
@@ -14,6 +14,10 @@
import sys
import os
import inspect
+import signal
+import time
+import argparse
+from copy import deepcopy
# Look under ext ford 3rd party dependencies
cmd_subfolder = os.path.realpath(os.path.abspath(os.path.join(os.path.split(inspect.getfile(inspect.currentframe()))[0], "ext")))
@@ -52,20 +56,24 @@ import cherrypy
import random
from cherrypy.lib import auth_digest
from sflib import SpiderFoot
+from sfdb import SpiderFootDb
from sfwebui import SpiderFootWebUi
+from sfscan import SpiderFootScanner
# 'Global' configuration options
# These can be overriden on a per-module basis, and some will
# be overridden from saved configuration settings stored in the DB.
sfConfig = {
'_debug': False, # Debug
+ '__logging': True, # Logging in general
'__blocknotif': False, # Block notifications
+ '_fatalerrors': False,
'_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
'_dnsserver': '', # Override the default resolver
'_fetchtimeout': 5, # number of seconds before giving up on a fetch
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
- '__version__': '2.12',
+ '__version__': '3.0',
'__database': 'spiderfoot.db',
'__webaddr': '127.0.0.1',
'__webport': 5001,
@@ -77,7 +85,8 @@ sfConfig = {
'_socks4user': '',
'_socks5pwd': '',
'_socks6dns': True,
- '_torctlport': 9051
+ '_torctlport': 9051,
+ '__logstdout': False
}
sfOptdescs = {
@@ -94,20 +103,59 @@ sfOptdescs = {
'_socks5pwd': "SOCKS Password. Valid only for SOCKS5 servers.",
'_socks6dns': "Resolve DNS through the SOCKS proxy? Has no affect when TOR is used: Will always be True.",
'_torctlport': "The port TOR is taking control commands on. This is necessary for SpiderFoot to tell TOR to re-circuit when it suspects anonymity is compromised.",
+ '_fatalerrors': "Abort the scan when modules encounter exceptions.",
'_modulesenabled': "Modules enabled for the scan." # This is a hack to get a description for an option not actually available.
}
+scanId = None
+dbh = None
+
+def handle_abort(signal, frame):
+ print "[*] Aborting..."
+ if scanId and dbh:
+ dbh.scanInstanceSet(scanId, None, None, "ABORTED")
+ sys.exit(-1)
+
if __name__ == '__main__':
- if len(sys.argv) > 1:
- (addr, port) = sys.argv[1].split(":")
- sfConfig['__webaddr'] = addr
- sfConfig['__webport'] = int(port)
+ # Legacy way to run the server
+ args = None
+ if (len(sys.argv) > 1 and ":" in sys.argv[1]) or len(sys.argv) == 1:
+ if len(sys.argv) > 1:
+ (addr, port) = sys.argv[1].split(":")
+ sfConfig['__webaddr'] = addr
+ sfConfig['__webport'] = int(port)
+ sfConfig['__logstdout'] = False
+ else:
+ p = argparse.ArgumentParser(description='SpiderFoot 3.0: Open Source Intelligence Automation.')
+ p.add_argument("-d", "--debug", action='store_true', help="Enable debug output.")
+ p.add_argument("-m", metavar="mod1,mod2,...", type=str, help="Modules to enable.")
+ p.add_argument("-M", "--modules", action='store_true', help="List available modules.")
+ p.add_argument("-s", metavar="TARGET", help="Target for the scan.")
+ p.add_argument("-t", metavar="type1,type2,...", type=str, help="Event types to enable.")
+ p.add_argument("-T", "--types", action='store_true', help="List available event types.")
+ p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default.")
+ p.add_argument("-n", action='store_true', help="Strip newlines from data.")
+ p.add_argument("-S", metavar="LENGTH", type=int, help="Maximum data length to display. By default, all data is shown.")
+ p.add_argument("-f", action='store_true', help="Filter out other event types that weren't requested with -t.")
+ p.add_argument("-F", metavar="FILTER", type=str, help="Filter out a set of event types.")
+ p.add_argument("-q", action='store_true', help="Disable logging.")
+ args = p.parse_args()
- sf = SpiderFoot(sfConfig)
- sfModules = dict()
+ sfConfig['__logstdout'] = True
+
+ if args.debug:
+ sfConfig['_debug'] = True
+ else:
+ sfConfig['_debug'] = False
+
+ if args.q:
+ sfConfig['__logging'] = False
+
+ sfModules = dict()
+ sft = SpiderFoot(sfConfig)
# Go through each module in the modules directory with a .py extension
- for filename in os.listdir(sf.myPath() + '/modules/'):
+ for filename in os.listdir(sft.myPath() + '/modules/'):
if filename.startswith("sfp_") and filename.endswith(".py"):
# Skip the module template and debugging modules
if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':
@@ -139,6 +187,124 @@ if __name__ == '__main__':
# Add descriptions of the global config options
sfConfig['__globaloptdescs__'] = sfOptdescs
+ sf = SpiderFoot(sfConfig)
+ dbh = SpiderFootDb(sfConfig)
+
+ if args:
+ if args.modules:
+ print "Modules available:"
+ for m in sorted(sfModules.keys()):
+ if "__" in m:
+ continue
+ print '{0:25} {1}'.format(m, sfModules[m]['descr'])
+ sys.exit(0)
+
+ if args.types:
+ print "Types available:"
+ typedata = dbh.eventTypes()
+ types = dict()
+ for r in typedata:
+ types[r[1]] = r[0]
+
+ for t in sorted(types.keys()):
+ print '{0:45} {1}'.format(t, types[t])
+ sys.exit(0)
+
+ if not args.s:
+ print "You must specify a target when running in scan mode. Try sf.py --help for guidance."
+ sys.exit(-1)
+
+ target = args.s
+ targetType = sf.targetType(args.s)
+
+ modlist = list()
+ if not args.t and not args.m:
+ print "WARNING: You didn't specify any modules or types, so all will be enabled."
+ for m in sfModules.keys():
+ if "__" in m:
+ continue
+ modlist.append(m)
+
+ signal.signal(signal.SIGINT, handle_abort)
+ # If the user is scanning by type..
+ # 1. Find modules producing that type
+ if args.t:
+ types = args.t
+ modlist = sf.modulesProducing(types)
+ newmods = deepcopy(modlist)
+ newmodcpy = deepcopy(newmods)
+ # 2. For each type those modules consume, get modules producing
+ while len(newmodcpy) > 0:
+ for etype in sf.eventsToModules(newmodcpy):
+ xmods = sf.modulesProducing([etype])
+ for mod in xmods:
+ if mod not in modlist:
+ modlist.append(mod)
+ newmods.append(mod)
+ newmodcpy = deepcopy(newmods)
+ newmods = list()
+
+ # Easier if scanning by module
+ if args.m:
+ modlist = args.m.split(",")
+
+ # Add sfp__stor_stdout to the module list
+ outputformat = "tab"
+ typedata = dbh.eventTypes()
+ types = dict()
+ for r in typedata:
+ types[r[1]] = r[0]
+
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_eventtypes'] = types
+ if args.f:
+ if args.f and not args.t:
+ print "You can only use -f with -t. Use --help for guidance."
+ sys.exit(-1)
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_showonlyrequested'] = True
+ if args.F:
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_requested'] = args.F.split(",")
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_showonlyrequested'] = True
+ if args.o:
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_format'] = args.o
+ if args.t:
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_requested'] = args.t.split(",")
+ if args.n:
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_stripnewline'] = True
+ if args.S:
+ sfConfig['__modules__']['sfp__stor_stdout']['opts']['_maxlength'] = args.S
+
+ modlist += ["sfp__stor_db", "sfp__stor_stdout"]
+
+ # Run the scan
+ if not args.q:
+ print "[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)
+ cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
+ scanId = sf.genScanInstanceGUID(target)
+
+ # Debug mode is a variable that gets stored to the DB, so re-apply it
+ if args.debug:
+ cfg['_debug'] = True
+ else:
+ cfg['_debug'] = False
+
+ t = SpiderFootScanner(target, target, targetType, scanId,
+ modlist, cfg, dict())
+ t.daemon = True
+ t.start()
+
+ while True:
+ info = dbh.scanInstanceGet(scanId)
+ if not info:
+ time.sleep(1)
+ continue
+ if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:
+ if not args.q:
+ print "[*] Scan completed with status " + info[5]
+ sys.exit(0)
+ time.sleep(1)
+
+ sys.exit(0)
+
# Start the web server so you can start looking at results
print "Starting web server at http://" + sfConfig['__webaddr'] + \
":" + str(sfConfig['__webport']) + sfConfig['__docroot'] + " ..."
diff --git a/sfcli.py b/sfcli.py
index 3f3a60f3..53d2dff3 100644
--- a/sfcli.py
+++ b/sfcli.py
@@ -35,7 +35,7 @@ class bcolors:
GREYBLUE_DARK = '\x1b[38;5;24m'
class SpiderFootCli(cmd.Cmd):
- version = "2.12"
+ version = "3.0"
pipecmd = None
output = None
modules = []
diff --git a/static/img/spiderfoot-icon.png b/static/img/spiderfoot-icon.png
index 11cc6b5c..706bc826 100644
Binary files a/static/img/spiderfoot-icon.png and b/static/img/spiderfoot-icon.png differ