Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

SpiderFoot CLI!

Steve0a8becc26103
diff --git a/sfcli.py b/sfcli.py
new file mode 100644
index 00000000..4eb3df20
--- /dev/null
+++ b/sfcli.py
@@ -0,0 +1,1186 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfcli
+# Purpose: Command Line Interface for SpiderFoot.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 03/05/2017
+# Copyright: (c) Steve Micallef 2017
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import cmd
+import sys
+import os
+import re
+import json
+import requests
+import shlex
+import codecs
+import time
+import readline
+import argparse
+from os.path import expanduser
+
+# Colors to make things purty
+class bcolors:
+ GREYBLUE = '\x1b[38;5;25m'
+ GREY = '\x1b[38;5;243m'
+ DARKRED = '\x1b[38;5;124m'
+ DARKGREEN = '\x1b[38;5;30m'
+ BOLD = '\033[1m'
+ ENDC = '\033[0m'
+ GREYBLUE_DARK = '\x1b[38;5;24m'
+
+class SpiderFootCli(cmd.Cmd):
+ version = "2.10"
+ pipecmd = None
+ output = None
+ prompt = "sf> "
+ nohelp = "[!] Unknown command '%s'."
+ knownscans = []
+ ownopts = {
+ "cli.debug": False,
+ "cli.silent": False,
+ "cli.color": True,
+ "cli.output": "pretty",
+ "cli.history": True,
+ "cli.history_file": "",
+ "cli.spool": False,
+ "cli.spool_file": "",
+ "cli.username": "",
+ "cli.password": "",
+ "cli.server_baseurl": "http://127.0.0.1:5001"
+ }
+
+ def dprint(self, msg, err=False, deb=False, plain=False, color=None):
+ cout = ""
+ sout = ""
+ pfx = ""
+ if err:
+ pfx = "[!]"
+ if self.ownopts['cli.color']:
+ col = bcolors.DARKRED
+ else:
+ pfx = "[*]"
+ if self.ownopts['cli.color']:
+ col = bcolors.DARKGREEN
+ if deb:
+ if not self.ownopts["cli.debug"]:
+ return
+ pfx = "[+]"
+ if self.ownopts['cli.color']:
+ col = bcolors.GREY
+
+ if color:
+ pfx = ""
+ col = color
+
+ if err or not self.ownopts["cli.silent"]:
+ if not plain or color:
+ cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + \
+ col + msg + bcolors.ENDC
+ # Never include color in the spool
+ sout = pfx + " " + msg
+ else:
+ cout = msg
+ sout = msg
+
+ print cout
+
+ if self.ownopts['cli.spool']:
+ f = codecs.open(self.ownopts['cli.spool_file'], "a", encoding="utf-8")
+ f.write(sout)
+ f.write('\n')
+ f.close()
+
+ # Shortcut commands
+ def do_debug(self, line):
+ """debug
+ Short-cut command for set cli.debug = 1"""
+ if self.ownopts['cli.debug']:
+ val = "0"
+ else:
+ val = "1"
+ return self.do_set("cli.debug = " + val)
+
+ def do_spool(self, line):
+ """spool
+ Short-cut command for set cli.spool = 1/0"""
+ if self.ownopts['cli.spool']:
+ val = "0"
+ else:
+ val = "1"
+ if self.ownopts['cli.spool_file']:
+ return self.do_set("cli.spool = " + val)
+ else:
+ self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")
+
+ def do_history(self, line):
+ """history [-l]
+ Short-cut command for set cli.history = 1/0. Add -l to just list the history."""
+ c = self.myparseline(line)
+
+ if '-l' in c[0]:
+ i = 0
+ while i < readline.get_current_history_length():
+ self.dprint(readline.get_history_item(i), plain=True)
+ i += 1
+ return
+ if self.ownopts['cli.history']:
+ val = "0"
+ else:
+ val = "1"
+ return self.do_set("cli.history = " + val)
+
+ # Run before all commands to handle history and spooling
+ def precmd(self, line):
+ if self.ownopts['cli.history'] and line != "EOF":
+ f = codecs.open(self.ownopts["cli.history_file"], "a", encoding="utf-8")
+ f.write(line)
+ f.write('\n')
+ f.close()
+ if self.ownopts['cli.spool']:
+ f = codecs.open(self.ownopts["cli.spool_file"], "a", encoding="utf-8")
+ f.write(self.prompt + line)
+ f.write('\n')
+ f.close()
+
+ return line
+
+ # Debug print
+ def ddprint(self, msg):
+ self.dprint(msg, deb=True)
+
+ # Error print
+ def edprint(self, msg):
+ self.dprint(msg, err=True)
+
+ # Print nice tables.
+ def pretty(self, data, titlemap=None):
+ out = list()
+ # Get the column titles
+ maxsize = dict()
+ if type(data[0]) == dict:
+ cols = data[0].keys()
+ else:
+ # for lists, use the index numbers as titles
+ cols = map(str, range(0, len(data[0])))
+
+ # Strip out columns that don't have titles
+ if titlemap:
+ nc = list()
+ for c in cols:
+ if c in titlemap:
+ nc.append(c)
+ cols = nc
+
+ spaces = 2
+ # Find the maximum column sizes
+ for r in data:
+ i = 0
+ for c in r:
+ if type(r) == list:
+ # we have list index
+ cn = str(i)
+ if type(c) == int:
+ v = str(c)
+ if type(c) == str:
+ v = unicode(c, errors='replace')
+ if type(c) == unicode:
+ v = c
+ else:
+ # we have a dict key
+ cn = c
+ v = r[c]
+ #print str(cn) + ", " + str(c) + ", " + str(v)
+ if len(v) > maxsize.get(cn, 0):
+ maxsize[cn] = len(v)
+ i += 1
+
+
+ # Adjust for long titles
+ if titlemap:
+ for c in maxsize:
+ if len(titlemap.get(c, c)) > maxsize[c]:
+ maxsize[c] = len(titlemap.get(c, c))
+
+ # Display the column titles
+ i = 0
+ for c in cols:
+ if titlemap:
+ t = titlemap.get(c, c)
+ else:
+ t = c
+ #out += t
+ out.append(t)
+ sdiff = maxsize[c] - len(t) + 1
+ #out += " " * spaces
+ out.append(" " * spaces)
+ if sdiff > 0 and i < len(cols)-1:
+ #out += " " * sdiff
+ out.append(" " * sdiff)
+ i += 1
+ #out += "\n"
+ out.append('\n')
+
+ # Then the separator
+ i = 0
+ for c in cols:
+ #out += "-" * ((maxsize[c]+spaces))
+ out.append("-" * ((maxsize[c]+spaces)))
+ if i < len(cols)-1:
+ #out += "+"
+ out.append("+")
+ i += 1
+ #out += "\n"
+ out.append("\n")
+
+ # Then the actual data
+ #ts = time.time()
+ for r in data:
+ i = 0
+ di = 0
+ tr = type(r)
+ for c in r:
+ if tr == list:
+ # we have list index
+ cn = str(i)
+ tc = type(c)
+ if tc == int:
+ v = str(c)
+ if tc == str:
+ v = unicode(c, errors='replace')
+ if tc == unicode:
+ v = c
+ else:
+ # we have a dict key
+ cn = c
+ v = r[c]
+ if cn not in cols:
+ i += 1
+ continue
+
+ out.append(v)
+ lv = len(v)
+ # there is a preceeding space if this is after the
+ # first column
+ # sdiff = number of spaces between end of word and |
+ if di == 0:
+ sdiff = (maxsize[cn] - lv) + spaces
+ else:
+ sdiff = (maxsize[cn] - lv) + spaces - 1
+ if di < len(cols)-1:
+ #out += " " * sdiff
+ out.append(" " * sdiff)
+ if di < len(cols)-1:
+ #out += "| "
+ out.append("| ")
+ di += 1
+ i += 1
+ #out += "\n"
+ out.append("\n")
+
+ #print "time: " + str(time.time() - ts)
+ #return ""
+ return ''.join(out)
+
+ # Make a request to the SpiderFoot server
+ def request(self, url, post=None):
+ #logging.basicConfig()
+ #logging.getLogger().setLevel(logging.DEBUG)
+ #requests_log = logging.getLogger("requests.packages.urllib3")
+ #requests_log.setLevel(logging.DEBUG)
+ #requests_log.propagate = True
+ try:
+ headers = {
+ "User-agent": "SpiderFoot-CLI/" + self.version
+ }
+
+ self.ddprint("Fetching: " + url)
+ self.ddprint("Posting: " + str(post))
+ if not post:
+ r = requests.get(url,
+ headers = headers,
+ auth = requests.auth.HTTPDigestAuth(
+ self.ownopts['cli.username'],
+ self.ownopts['cli.password']
+ )
+ )
+ else:
+ r = requests.post(url,
+ headers = headers,
+ auth = requests.auth.HTTPDigestAuth(
+ self.ownopts['cli.username'],
+ self.ownopts['cli.password']
+ ),
+ data = post
+ )
+ self.ddprint("Response: " + str(r))
+ if r.status_code == requests.codes.ok:
+ return r.text
+ else:
+ r.raise_for_status()
+ except BaseException as e:
+ self.edprint("Failed communicating with server: " + str(e))
+ return None
+
+ def emptyline(self):
+ return
+
+ def completedefault(self, text, line, begidx, endidx):
+ return []
+
+ # Parse the command line, returns a list of lists:
+ # sf> scans "blahblah test" | top 10 | grep foo ->
+ # [[ 'blahblah test' ], [[ 'top', '10' ], [ 'grep', 'foo']]]
+ def myparseline(self, cmdline, replace=True):
+ ret = [list(),list()]
+ s = shlex.split(cmdline.decode('string_escape'))
+ for c in s:
+ if c == '|':
+ break
+ if replace and c.startswith("$") and c in self.ownopts:
+ ret[0].append(self.ownopts[c])
+ else:
+ ret[0].append(c)
+
+ if s.count('|') == 0:
+ return ret
+
+ # Handle any pipe commands at the end
+ ret[1] = list()
+ i = 0
+ ret[1].append(list())
+ for t in s[(s.index('|')+1):]:
+ if t == '|':
+ i += 1
+ ret[1].append(list())
+ continue
+ # Replace variables
+ if t.startswith("$") and t in self.ownopts:
+ ret[1][i].append(self.ownopts[t])
+ else:
+ ret[1][i].append(t)
+
+ return ret
+
+ # Send the command output to the user, proessing the pipes
+ # that may have been used.
+ def send_output(self, data, cmd, titles=None, total=True, raw=False):
+ totalrec = 0
+ out = None
+ try:
+ if not raw:
+ j = json.loads(data)
+ totalrec = len(j)
+ else:
+ j = data
+ except BaseException as e:
+ self.edprint("Unable to parse data from server: " + str(e))
+ return
+
+ if not raw:
+ if self.ownopts['cli.output'] == "json":
+ out = json.dumps(j, indent=4, separators=(',', ': '))
+
+ if self.ownopts['cli.output'] == "pretty":
+ out = self.pretty(j, titlemap=titles)
+
+ if not out:
+ self.edprint("Unknown output format '" + self.ownopts['cli.output'] + "'.")
+ return
+ else:
+ out = data
+
+ c = self.myparseline(cmd)
+
+ # If no pipes, just disply the output
+ if len(c[1]) == 0:
+ self.dprint(out, plain=True)
+ if total:
+ self.dprint("Total records: " + str(totalrec))
+ return
+
+ for pc in c[1]:
+ newout = ""
+ if len(pc) == 0:
+ self.edprint("Invalid syntax.")
+ return
+ pipecmd = pc[0]
+ pipeargs = " ".join(pc[1:])
+ if pipecmd not in [ "str", "regex", "file", "grep", "top", "last" ]:
+ self.edprint("Unrecognised pipe command.")
+ return
+
+ if pipecmd == "regex":
+ p = re.compile(pipeargs, re.IGNORECASE)
+ for r in out.split("\n"):
+ if re.match(p, r.strip()):
+ newout += r + "\n"
+
+ if pipecmd == "str" or pipecmd == "grep":
+ for r in out.split("\n"):
+ if pipeargs.lower() in r.strip().lower():
+ newout += r + "\n"
+
+ if pipecmd == "top":
+ if not pipeargs.isdigit():
+ self.edprint("Invalid syntax.")
+ return
+ newout = "\n".join(out.split("\n")[0:int(pipeargs)])
+
+ if pipecmd == "last":
+ if not pipeargs.isdigit():
+ self.edprint("Invalid syntax.")
+ return
+ tot = len(out.split("\n"))
+ i = tot - int(pipeargs)
+ newout = "\n".join(out.split("\n")[i:])
+
+ if pipecmd == "file":
+ try:
+ f = codecs.open(pipeargs, "w", encoding="utf-8")
+ f.write(out)
+ f.close()
+ except BaseException as e:
+ self.edprint("Unable to write to file: " + str(e))
+ return
+ self.dprint("Successfully wrote to file '" + str(pipeargs) + "'.")
+ return
+
+ out = newout
+
+ self.dprint(newout, plain=True)
+
+ # Run SQL against the DB.
+ def do_query(self, line):
+ """query <SQL query>
+ Run an <SQL query> against the database."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+ query = ' '.join(c[0])
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/query",
+ post={"query": query})
+ if not d:
+ return
+ j = json.loads(d)
+ if j[0] == "ERROR":
+ self.edprint("Error running your query: " + j[1])
+ return
+ self.send_output(d, line)
+
+ # Ping the server.
+ def do_ping(self, line):
+ """ping
+ Ping the SpiderFoot server to ensure it's responding."""
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/ping")
+ if not d:
+ return
+
+ s = json.loads(d)
+ if s[0] == "SUCCESS":
+ self.dprint("Server " + self.ownopts['cli.server_baseurl'] + " responding.")
+ else:
+ self.dprint("Something odd happened: " + str(d))
+
+ if s[1] != self.version:
+ self.edprint("Server and CLI version are not the same (" + s[1] + \
+ " / " + self.version + "). This could lead to unpredictable results!")
+
+ # List all SpiderFoot modules.
+ def do_modules(self, line):
+ """modules
+ List all available modules and their descriptions."""
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/modules")
+ if not d:
+ return
+ self.send_output(d, line, titles={"name": "Module name",
+ "descr": "Description"})
+
+ # List all SpiderFoot data element types.
+ def do_types(self, line):
+ """types
+ List all available element types and their descriptions."""
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/eventtypes")
+ if not d:
+ return
+ self.send_output(d, line, titles={ "1": "Element description",
+ "0": "Element name"})
+
+ # Load commands from a file.
+ def do_load(self, line):
+ """load <file>
+ Execute SpiderFoot CLI commands found in <file>."""
+ pass
+
+ # Get scan info and config.
+ def do_scaninfo(self, line):
+ """scaninfo <sid> [-c]
+ Get status information for scan ID <sid>, optionally also its configuration if -c is supplied."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+
+ sid = c[0][0]
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanopts?id=" + sid)
+ if not d:
+ return
+ j = json.loads(d)
+ if len(j) == 0:
+ self.dprint("No such scan exists.")
+ return
+
+ out = "Name: " + j['meta'][0] + "\n"
+ out += "ID: " + sid + "\n"
+ out += "Target: " + j['meta'][1] + "\n"
+ out += "Started: " + j['meta'][3] + "\n"
+ out += "Completed: " + j['meta'][4] + "\n"
+ out += "Status: " + j['meta'][5] + "\n"
+
+ if "-c" in c[0]:
+ out += "Configuration:\n"
+ for k in j['config']:
+ out += " " + k + " = " + j['config'][k] + "\n"
+
+ self.send_output(out, line, total=False, raw=True)
+
+ # List scans.
+ def do_scans(self, line):
+ """scans [-x]
+ List all scans, past and present. -x for extended view."""
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlist")
+ if not d:
+ return
+ j = json.loads(d)
+ if len(j) == 0:
+ self.dprint("No scans exist.")
+ return
+
+ c = self.myparseline(line)
+ titles = dict()
+ if "-x" in c[0]:
+ titles = {"0": "ID",
+ "1": "Name",
+ "2": "Target",
+ "3": "Created",
+ "4": "Started",
+ "5": "Finished",
+ "6": "Status",
+ "7": "Total Elements"
+ }
+ else:
+ titles = {
+ "0": "ID",
+ "2": "Target",
+ "6": "Status",
+ "7": "Total Elements"
+ }
+
+ self.send_output(d, line, titles=titles)
+
+ # Show the data from a scan.
+ def do_data(self, line):
+ """data <sid> [type] [-x] [-u]
+ Get the scan data for scan ID <sid> and optionally the element type [type] (e.g. EMAILADDR),
+ [type]. Use -x for extended format. Use -u for a unique set of results."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+
+ post = { "id": c[0][0] }
+
+ if "-t" in c[0]:
+ post["eventType"] = c[0][c[0].index("-t")+1]
+ else:
+ post["eventType"] = "ALL"
+
+ if "-u" in c[0]:
+ url = self.ownopts['cli.server_baseurl'] + "/scaneventresultsunique"
+ col = 0
+ else:
+ url = self.ownopts['cli.server_baseurl'] + "/scaneventresults"
+ col = 1
+
+ d = self.request(url, post=post)
+ if not d:
+ return
+ j = json.loads(d)
+ if len(j) < 1:
+ self.dprint("No results.")
+ return
+
+ titles={
+ str(col): "Data"
+ }
+ if "-x" in c[0]:
+ titles["0"] = "Last Seen"
+ titles["3"] = "Module"
+ titles["2"] = "Source Data"
+
+ self.send_output(d, line, titles=titles)
+
+ # Show logs.
+ def do_logs(self, line):
+ """logs <sid> [-l count] [-w]
+ Show the most recent [count] logs for a given scan ID, <sid>.
+ If no count is supplied, all logs are given.
+ If -w is supplied, logs will be streamed to the console until Ctrl-C is entered."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+ sid = c[0][0]
+ limit = None
+ if "-l" in c[0]:
+ limit = c[0][c[0].index("-l")+1]
+
+ if "-w" not in c[0]:
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={ 'id': sid, 'limit': limit })
+ if not d:
+ return
+ j = json.loads(d)
+ if len(j) < 1:
+ self.dprint("No results.")
+ return
+
+ self.send_output(d, line, titles={"0": "Generated",
+ "1": "Type",
+ "2": "Source",
+ "3": "Message"
+ }
+ )
+ return
+ else:
+ # Get the rowid of the latest log message
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'limit': '1'})
+ if not d:
+ return
+ j = json.loads(d)
+ if len(j) < 1:
+ self.dprint("No logs (yet?).")
+ return
+ else:
+ rowid = j[0][4]
+
+ try:
+ if not limit:
+ limit = 10
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'reverse': '1', 'rowId': rowid-int(limit)})
+ if not d:
+ return
+ j = json.loads(d)
+ for r in j:
+ self.send_output(str(r), line, total=False, raw=True)
+ if r[2] == "ERROR":
+ self.edprint(r[1] + ": " + r[3])
+ else:
+ self.dprint(r[1] + ": " + r[3])
+
+ while True:
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'reverse': '1', 'rowId': rowid})
+ if not d:
+ return
+ j = json.loads(d)
+ for r in j:
+ if r[2] == "ERROR":
+ self.edprint(r[1] + ": " + r[3])
+ else:
+ self.dprint(r[1] + ": " + r[3])
+
+ rowid = str(r[4])
+ time.sleep(0.5)
+ except KeyboardInterrupt:
+ return
+
+ # Start a new scan.
+ def do_start(self, line):
+ """start <target> <-m module1,module2,...>|<-t type1,type2>|<-u "Use case"> [-n name]
+ Start a scan against <target> using modules or looking for types or by Use case ("all",
+ "investigate", "passive" and "footprint").
+ Scan be be optionally named [name], without a name the target will be used.
+ """
+ mods = ""
+ types = ""
+ usecase = ""
+ c = self.myparseline(line)
+ if len(c[0]) < 3:
+ self.edprint("Invalid syntax.")
+ return
+
+ if "-m" in c[0]:
+ mods = c[0][c[0].index("-m")+1]
+
+ if "-t" in c[0]:
+ # Scan by type
+ types = c[0][c[0].index("-t")+1]
+
+ if "-u" in c[0]:
+ # Scan by use case
+ usecase = c[0][c[0].index("-u")+1]
+
+ if not mods and not types and not usecase:
+ self.edprint("Invalid syntax.")
+ return
+
+ target = c[0][0]
+ if "-n" in c[0]:
+ title = c[0][c[0].index("-n")+1]
+ else:
+ title = target
+
+ post = { "scanname": title,
+ "scantarget": target,
+ "modulelist": mods,
+ "typelist": types,
+ "usecase": usecase,
+ "cli": "1"
+ }
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",
+ post=post)
+ if not d:
+ return
+
+ s = json.loads(d)
+ if s[0] == "SUCCESS":
+ self.dprint("Successfully initiated scan.")
+ self.dprint("Scan ID: " + s[1])
+ else:
+ self.dprint("Unable to start scan: " + str(s[1]))
+
+ # Stop a running scan.
+ def do_stop(self, line):
+ """stop <sid>
+ Abort the running scan with scan ID, <sid>."""
+ c = self.myparseline(line)
+ try:
+ id = c[0][0]
+ except BaseException as e:
+ self.edprint("Invalid syntax.")
+ return
+
+ d = self.request(self.ownopts['cli.server_baseurl'] + \
+ "/stopscan?id=" + id + "&cli=1")
+ if not d:
+ return
+
+ s = json.loads(d)
+ if s[0] == "SUCCESS":
+ self.dprint("Successfully requested scan to stop. This could take some minutes to complete.")
+ else:
+ self.dprint("Unable to stop scan: " + str(s[1]))
+
+ # Search for data, alias to find
+ def do_search(self, line):
+ """search (look up 'find')
+ """
+ return self.do_find(line)
+
+ # Search for data
+ def do_find(self, line):
+ """find "<string|/regex/>" <[-s sid]|[-t type]> [-x]
+ Search for string/regex, limited to the scope of either a scan ID or event type. -x for extended format."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+
+ val = c[0][0]
+ sid = None
+ etype = None
+ if "-t" in c[0]:
+ etype = c[0][c[0].index("-t")+1]
+ if "-s" in c[0]:
+ sid = c[0][c[0].index("-s")+1]
+
+ titles={"0": "Last Seen",
+ "1": "Data",
+ "3": "Module"
+ }
+ if "-x" in c[0]:
+ titles["2"] = "Source Data"
+
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/search",
+ post={'value': val, 'id': sid, 'eventType': etype})
+ if not d:
+ return
+ j = json.loads(d)
+
+ if not j:
+ self.dprint("No results found.")
+ return
+ if len(j) < 1:
+ self.dprint("No results found.")
+ return
+
+ self.send_output(d, line, titles)
+
+ # Summary of a scan
+ def do_summary(self, line):
+ """summary <sid> [-t]
+ Summarise the results for a scan ID, <sid>. -t to only show the element types."""
+ c = self.myparseline(line)
+ if len(c[0]) < 1:
+ self.edprint("Invalid syntax.")
+ return
+
+ sid = c[0][0]
+
+ if "-t" in c[0]:
+ titles = {"0": "Element Type"}
+ else:
+ titles={"0": "Element Type",
+ "1": "Element Description",
+ "2": "Last Seen",
+ "3": "Total",
+ "4": "Unique"
+ }
+
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/scansummary?id=" + sid + "&by=type")
+ if not d:
+ return
+ j = json.loads(d)
+
+ if not j:
+ self.dprint("No results found.")
+ return
+ if len(j) < 1:
+ self.dprint("No results found.")
+ return
+
+ self.send_output(d, line, titles, total=False)
+
+ # Delete a scan
+ def do_delete(self, line):
+ """delete <sid>
+ Delete a scan with scan ID, <sid>."""
+ c = self.myparseline(line)
+ try:
+ id = c[0][0]
+ except BaseException as e:
+ self.edprint("Invalid syntax.")
+ return
+
+ d = self.request(self.ownopts['cli.server_baseurl'] + \
+ "/scandelete?confirm=1&raw=1&id=" + id)
+ if not d:
+ return
+
+ s = json.loads(d)
+ if s[0] == "SUCCESS":
+ self.dprint("Successfully deleted scan.")
+ else:
+ self.dprint("Something odd happened: " + str(s[1]))
+
+ # Override the default help
+ def print_topics(self, header, cmds, cmdlen, maxcol):
+ if not cmds:
+ return
+
+ helpmap = [
+ ["help [command]", "This help output. Specify 'command' for more detailed help."],
+ ["debug", "Enable/Disable debug output."],
+ ["clear", "Clear the screen."],
+ ["history [-l]", "Enable/Disable/List command history."],
+ ["spool", "Enable/Disable spooling output."],
+ ["shell", "Execute a shell command."],
+ ["exit", "Exit the SpiderFoot CLI (won't impact running scans)."],
+ ["ping", "Test connectivity to the SpiderFoot server."],
+ ["modules", "List available modules."],
+ ["types", "List available data types."],
+ ["set [[$]opt] [= val]", "Set variables and configuration settings."],
+ ["scans [-x]", "List all scans that have been run or are running."],
+ ["start <target> [-m m1,...] [-t t1,...] [-u name] [-n name]", "Start a new scan."],
+ ["stop <sid>", "Stop a scan."],
+ ["delete <sid>", "Delete a scan."],
+ ["scaninfo <sid> [-c]", "Scan information."],
+ ["data <sid> [type] [-x] [-u]", "Show data from a scan's results."],
+ ["summary <sid> [-t]", "Scan result summary."],
+ ["find <string|/regex/> [-s sid] [-t type] [-x]", "Search for data within scan results."],
+ ["query <SQL query>", "Run SQL against the SpiderFoot SQLite database."],
+ ["logs <sid> [-l count] [-w]", "View/watch logs from a scan."]
+ ]
+
+ self.send_output(json.dumps(helpmap), "", titles={"0": "Command", "1": "Description"},
+ total=False)
+
+ # Get/Set configuration
+ def do_set(self, line):
+ """set [opt [= <val>]]
+ Set a configuration variable in SpiderFoot."""
+
+ c = self.myparseline(line, replace=False)
+ cfg = None
+ val = None
+
+ if len(c[0]) > 0:
+ cfg = c[0][0]
+
+ if len(c[0]) > 2:
+ try:
+ val = c[0][2]
+ except BaseException as e:
+ self.edprint("Invalid syntax.")
+ return
+
+ # Get the server-side config
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/optsraw")
+ if not d:
+ return
+ j = list()
+ serverconfig = dict()
+ token = ""
+ if not d:
+ self.edprint("Unable to obtain SpiderFoot server-side config.")
+ else:
+ j = json.loads(d)
+ if j[0] == "ERROR":
+ self.edprint("Error fetching SpiderFoot server-side config.")
+ return
+ else:
+ serverconfig = j[1]['data']
+ token = j[1]['token']
+
+ self.ddprint(str(serverconfig))
+
+ if not cfg or not val:
+ ks = self.ownopts.keys()
+ ks.sort()
+ output = list()
+ for k in ks:
+ c = self.ownopts[k]
+ if type(c) == bool:
+ if c:
+ c = "True"
+ else:
+ c = "False"
+
+ if not cfg:
+ output.append({'opt': k, 'val': c})
+ else:
+ if cfg == k:
+ self.dprint(k + " = " + c, plain=True)
+
+ for k in sorted(serverconfig.keys()):
+ if type(serverconfig[k]) == list:
+ serverconfig[k] = ','.join(serverconfig[k])
+ if type(serverconfig[k]) == bool:
+ if serverconfig[k]:
+ serverconfig[k] = "True"
+ else:
+ serverconfig[k] = "False"
+
+ if not cfg:
+ output.append({'opt': k, 'val': str(serverconfig[k])})
+ else:
+ if cfg == k:
+ self.dprint(k + " = " + str(serverconfig[k]), plain=True)
+
+ if len(output) > 0:
+ self.send_output(json.dumps(output), line,
+ { 'opt': "Option", 'val': "Value"},
+ total=False)
+ return
+
+ if val:
+ # Local CLI config
+ if cfg in self.ownopts or cfg.startswith('$'):
+ if not cfg.startswith('$'):
+ if type(self.ownopts[cfg]) == bool:
+ if val.lower() == "false" or val == "0":
+ val = False
+ else:
+ val = True
+
+ self.ownopts[cfg] = val
+ self.dprint(cfg + " set to " + str(val))
+ return
+ # Server-side config
+ else:
+ # submit all non-CLI vars to the SF server
+ confdata = dict()
+ found = False
+ for k in serverconfig:
+ if k == cfg:
+ serverconfig[k] = val
+ found = True
+
+ if not found:
+ self.edprint("Variable not found, so not set.")
+ return
+
+ # Sanitize the data before sending it to the server
+ for k in serverconfig:
+ optstr = ":".join(k.split(".")[1:])
+ if type(serverconfig[k]) == bool:
+ if serverconfig[k] == "false" or serverconfig[k] == "0":
+ confdata[optstr] = "0"
+ else:
+ confdata[optstr] = "1"
+ if type(serverconfig[k]) == list:
+ # If set by the user, it must already be a
+ # string, not a list
+ confdata[optstr] = ','.join(serverconfig[k])
+ if type(serverconfig[k]) == int:
+ confdata[optstr] = str(serverconfig[k])
+ if type(serverconfig[k]) in [ str, unicode ]:
+ confdata[optstr] = serverconfig[k]
+
+ self.ddprint(str(confdata))
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/savesettingsraw",
+ post={'token': token,'allopts': json.dumps(confdata)})
+ j = list()
+ if not d:
+ self.edprint("Unable to set SpiderFoot server-side config.")
+ return
+ else:
+ j = json.loads(d)
+ if j[0] == "ERROR":
+ self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))
+ return
+ self.dprint(cfg + " set to " + str(val))
+ return
+
+ if cfg not in self.ownopts:
+ self.edprint("Variable not found, so not set. Did you mean to use a $ variable?")
+ return
+
+ # Execute a shell command locally and return the output
+ def do_shell(self, line):
+ """shell
+ Run a shell command locally."""
+ self.dprint("Running shell command:" + str(line))
+ self.dprint(os.popen(line).read(), plain=True)
+
+ def do_clear(self, line):
+ """clear
+ Clear the screen."""
+ sys.stderr.write("\x1b[2J\x1b[H")
+
+ # Exit the CLI
+ def do_exit(self, line):
+ """exit
+ Exit the SpiderFoot CLI."""
+ return True
+
+ # Ctrl-D
+ def do_EOF(self, line):
+ """EOF (Ctrl-D)
+ Exit the SpiderFoot CLI."""
+ print "\n",
+ return True
+
+if __name__ == "__main__":
+ p = argparse.ArgumentParser(description='SpiderFoot: Open Source Intelligence Automation.')
+ p.add_argument("-d", "--debug", help="Enable debug output.", action='store_true')
+ p.add_argument("-s", metavar="URL", type=str, help="Connect to SpiderFoot server on URL. By default, a connection to http://127.0.0.1:5001 will be attempted.")
+ p.add_argument("-u", metavar="USER", type=str, help="Username to authenticate to SpiderFoot server.")
+ p.add_argument("-p", metavar="PASS", type=str, help="Password to authenticate to SpiderFoot server. Consider using -g PASSFILE instead so that your password isn't visible in your shell history or in process lists!")
+ p.add_argument("-P", metavar="PASSFILE", type=str, help="File oontaining password to authenticate to SpiderFoot server. Ensure permissions on the file are set appropriately!")
+ p.add_argument("-e", metavar="FILE", type=str, help="Execute commands from FILE.")
+ p.add_argument("-l", metavar="FILE", type=str, help="Log command history to FILE. By default, history is stored to ~/.spiderfoot_history unless disabled with -n.")
+ p.add_argument("-n", action='store_true', help="Disable history logging.")
+ p.add_argument("-o", metavar="FILE", type=str, help="Spool commands and output to FILE.")
+ p.add_argument("-q", help="Silent output, only errors reported.", action='store_true')
+ p.add_argument("-k", help="Turn off color-coded output.", action='store_true')
+ p.add_argument("-b", "-v", help="Print the banner w/ version and exit.", action='store_true')
+
+ args = p.parse_args()
+
+ # Load commands from a file
+ if args.e:
+ try:
+ cin = open(args.e, "r")
+ except BaseException as e:
+ print "Unable to open " + args.e + ":" + " (" + str(e) + ")"
+ sys.exit(-1)
+ else:
+ cin = sys.stdin
+ s = SpiderFootCli(stdin=cin)
+ s.identchars += "$"
+
+ # Map command-line to config
+ if args.u:
+ s.ownopts['cli.username'] = args.u
+ if args.p:
+ s.ownopts['cli.password'] = args.p
+ if args.P:
+ try:
+ pf = open(args.P, "r")
+ s.ownopts['cli.password'] = pf.readlines()[0].strip('\n')
+ pf.close()
+ except BaseException as e:
+ print "Unable to open " + args.P + ":" + " (" + str(e) + ")"
+ sys.exit(-1)
+ if args.k:
+ s.ownopts['cli.color'] = False
+ if args.s:
+ s.ownopts['cli.server_baseurl'] = args.s
+ if args.debug:
+ s.ownopts['cli.debug'] = True
+ if args.q:
+ s.ownopts['cli.silent'] = True
+ if args.n:
+ s.ownopts['cli.history'] = False
+ if args.l:
+ s.ownopts['cli.history_file'] = args.l
+ else:
+ try:
+ s.ownopts['cli.history_file'] = expanduser("~") + "/.spiderfoot_history"
+ except BaseException as e:
+ s.ownopts['cli.history_file'] = ".spiderfoot_history"
+ if args.o:
+ s.ownopts['cli.spool'] = True
+ s.ownopts['cli.spool_file'] = args.o
+
+ if args.e:
+ try:
+ s.use_rawinput = False
+ s.prompt = ""
+ s.cmdloop()
+ finally:
+ cin.close()
+ sys.exit(0)
+
+ if not args.q:
+ s = SpiderFootCli()
+ s.dprint("\n\
+ _________ .__ .___ ___________ __ \n\
+ / _____/_____ |__| __| _/__________\\_ _____/___ _____/ |_ \n\
+ \_____ \\\\____ \| |/ __ |/ __ \\_ __ \\ __)/ _ \ / _ \\ __\\\n\
+ / \\ |_> > / /_/ \\ ___/| | \\/ \\( <_> | <_> ) | \n\
+/_______ / __/|__\\____ |\\___ >__| \\___ / \\____/ \\____/|__| \n\
+ \\/|__| \\/ \\/ \\/ \n\
+ Open Source Intelligence Automation.", plain=True,
+ color=bcolors.GREYBLUE)
+ s.dprint("\
+ by Steve Micallef | @binarypool\n", plain=True,
+ color=bcolors.GREYBLUE_DARK)
+ s.dprint("Version " + s.version + ".")
+ if args.b:
+ sys.exit(0)
+
+ # Test connectivity to the server
+ s.do_ping("")
+
+ if not args.n:
+ try:
+ f = codecs.open(s.ownopts['cli.history_file'], "r", encoding="utf-8")
+ for line in f.readlines():
+ readline.add_history(line.strip())
+ s.dprint("Loaded previous command history.")
+ except BaseException as e:
+ pass
+
+ s.dprint("Type 'help' or '?'.")
+ s.cmdloop()
diff --git a/sfwebui.py b/sfwebui.py
index bc454dc0..e12059a4 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -16,6 +16,7 @@ import csv
import time
import random
import re
+from operator import itemgetter
from copy import deepcopy
from mako.lookup import TemplateLookup
from mako.template import Template
@@ -236,6 +237,8 @@ class SpiderFootWebUi:
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
+ if not meta:
+ return json.dumps([])
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
@@ -398,21 +401,45 @@ class SpiderFootWebUi:
opts.exposed = True
+ # Settings
+ def optsraw(self):
+ ret = dict()
+ self.token = random.randint(0, 99999999)
+ for opt in self.config:
+ if opt.startswith('__'):
+ if opt == '__modules__':
+ for mod in sorted(self.config['__modules__'].keys()):
+ for mo in sorted(self.config['__modules__'][mod]['opts'].keys()):
+ if mo.startswith("_"):
+ continue
+ ret["module." + mod + "." + mo] = self.config['__modules__'][mod]['opts'][mo]
+ continue
+ ret["global." + opt] = self.config[opt]
+ return json.dumps(['SUCCESS', {'token': self.token, 'data': ret}])
+
+ optsraw.exposed = True
+
# Generic error, but not exposed as not called directly
def error(self, message):
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message=message, docroot=self.docroot)
# Delete a scan
- def scandelete(self, id, confirm=None):
+ def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
- return self.error("Scan ID not found.")
+ if not raw:
+ return self.error("Scan ID not found.")
+ else:
+ return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
- raise cherrypy.HTTPRedirect("/")
+ if not raw:
+ raise cherrypy.HTTPRedirect("/")
+ else:
+ return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], names=list(), ids=list(),
@@ -447,8 +474,8 @@ class SpiderFootWebUi:
# Save settings, also used to completely reset them to default
def savesettings(self, allopts, token):
- if str(token) != str(self.token):
- return self.error("Invalid token (" + str(self.token) + ").")
+# if str(token) != str(self.token):
+# return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
@@ -479,6 +506,37 @@ class SpiderFootWebUi:
savesettings.exposed = True
+ # Save settings, also used to completely reset them to default
+ def savesettingsraw(self, allopts, token):
+ if str(token) != str(self.token):
+ return json.dumps(["ERROR", "Invalid token (" + str(self.token) + ")."])
+
+ try:
+ dbh = SpiderFootDb(self.config)
+ # Reset config to default
+ if allopts == "RESET":
+ dbh.configClear() # Clear it in the DB
+ self.config = deepcopy(self.defaultConfig) # Clear in memory
+ else:
+ useropts = json.loads(allopts)
+ cleanopts = dict()
+ for opt in useropts.keys():
+ cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
+
+ currentopts = deepcopy(self.config)
+
+ # Make a new config where the user options override
+ # the current system config.
+ sf = SpiderFoot(self.config)
+ self.config = sf.configUnserialize(cleanopts, currentopts)
+ dbh.configSet(sf.configSerialize(currentopts))
+ except Exception as e:
+ return json.dumps(["ERROR", "Processing one or more of your inputs failed: " + str(e)])
+
+ return json.dumps(["SUCCESS", ""])
+
+ savesettingsraw.exposed = True
+
# Set a bunch of results (hashes) as false positive
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
@@ -521,37 +579,49 @@ class SpiderFootWebUi:
def eventtypes(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
+ ret = list()
- return json.dumps(types)
+ for r in types:
+ ret.append([r[1], r[0]])
+
+ ret = sorted(ret, key=itemgetter(0))
+
+ return json.dumps(ret)
eventtypes.exposed = True
# For the CLI to fetch a list of modules.
def modules(self):
modinfo = self.config['__modules__'].keys()
- return json.dumps(modinfo)
+ modinfo.sort()
+ ret = list()
+ for m in modinfo:
+ if "__" in m:
+ continue
+ ret.append({'name': m, 'descr': self.config['__modules__'][m]['descr']})
+ return json.dumps(ret)
modules.exposed = True
# For the CLI to test connectivity to this server.
def ping(self):
- return json.dumps(["SUCCESS", ""])
+ return json.dumps(["SUCCESS", self.config['__version__']])
ping.exposed = True
# For the CLI to run queries against the database.
def query(self, query):
data = None
- dbh = SpiderFootDb(cfg)
- if not query.startswith("SELECT"):
- return json_dumps(["ERROR", "Non-SELECTs are unpredictable and not recommended."])
+ dbh = SpiderFootDb(self.config)
+ if not query.lower().startswith("select"):
+ return json.dumps(["ERROR", "Non-SELECTs are unpredictable and not recommended."])
try:
- ret = dbh.execute(query)
+ ret = dbh.dbh.execute(query)
data = ret.fetchall()
except BaseException as e:
- return json_dumps(["ERROR", str(e)])
+ return json.dumps(["ERROR", str(e)])
- return json_dumps(data)
+ return json.dumps(data)
query.exposed = True
@@ -702,7 +772,7 @@ class SpiderFootWebUi:
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
else:
- return json.dumps(["ERROR", "Scan is still running."])
+ return json.dumps(["ERROR", "Scan doesn't appear to be running."])
if globalScanStatus.getStatus(id) == "ABORTED":
if not cli:
@@ -733,14 +803,14 @@ class SpiderFootWebUi:
#
# Scan log data
- def scanlog(self, id, limit=None):
+ def scanlog(self, id, limit=None, rowId=None, reverse=None):
dbh = SpiderFootDb(self.config)
- data = dbh.scanLogs(id, limit)
+ data = dbh.scanLogs(id, limit, rowId, reverse)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))
retdata.append([generated, row[1], row[2],
- cgi.escape(unicode(row[3], errors='replace'))])
+ cgi.escape(unicode(row[3], errors='replace')), row[4]])
return json.dumps(retdata)
scanlog.exposed = True