Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfcli: Cleanup (#1302)

bcolesb7eecdc5bc4e
diff --git a/sfcli.py b/sfcli.py
index e9677781..200d7bb1 100755
--- a/sfcli.py
+++ b/sfcli.py
@@ -355,7 +355,7 @@ class SpiderFootCli(cmd.Cmd):
return None
if not isinstance(url, str):
- self.edprint("Invalid request URL: %s" % url)
+ self.edprint(f"Invalid request URL: {url}")
return None
# logging.basicConfig()
@@ -363,14 +363,13 @@ class SpiderFootCli(cmd.Cmd):
# requests_log = logging.getLogger("requests.packages.urllib3")
# requests_log.setLevel(logging.DEBUG)
# requests_log.propagate = True
- try:
- headers = {
- "User-agent": "SpiderFoot-CLI/" + self.version,
- "Accept": "application/json"
- }
+ headers = {
+ "User-agent": "SpiderFoot-CLI/" + self.version,
+ "Accept": "application/json"
+ }
- self.ddprint("Fetching: " + url)
- self.ddprint("Posting: " + str(post))
+ try:
+ self.ddprint(f"Fetching: {url}")
if not post:
r = requests.get(
url,
@@ -382,6 +381,7 @@ class SpiderFootCli(cmd.Cmd):
)
)
else:
+ self.ddprint(f"Posting: {post}")
r = requests.post(
url,
headers=headers,
@@ -392,13 +392,12 @@ class SpiderFootCli(cmd.Cmd):
),
data=post
)
- self.ddprint("Response: " + str(r))
+ self.ddprint(f"Response: {r}")
if r.status_code == requests.codes.ok: # pylint: disable=no-member
return r.text
- else:
- r.raise_for_status()
+ r.raise_for_status()
except BaseException as e:
- self.edprint("Failed communicating with server: " + str(e))
+ self.edprint(f"Failed communicating with server: {e}")
return None
def emptyline(self):
@@ -416,7 +415,12 @@ class SpiderFootCli(cmd.Cmd):
if not cmdline:
return ret
- s = shlex.split(cmdline)
+ try:
+ s = shlex.split(cmdline)
+ except Exception as e:
+ self.edprint(f"Error parsing command: {e}")
+ return ret
+
for c in s:
if c == '|':
break
@@ -436,9 +440,8 @@ class SpiderFootCli(cmd.Cmd):
if t == '|':
i += 1
ret[1].append(list())
- continue
# Replace variables
- if t.startswith("$") and t in self.ownopts:
+ elif t.startswith("$") and t in self.ownopts:
ret[1][i].append(self.ownopts[t])
else:
ret[1][i].append(t)
@@ -448,19 +451,21 @@ class SpiderFootCli(cmd.Cmd):
# Send the command output to the user, processing the pipes
# that may have been used.
def send_output(self, data, cmd, titles=None, total=True, raw=False):
- totalrec = 0
out = None
try:
- if not raw:
+ if raw:
+ j = data
+ totalrec = 0
+ else:
j = json.loads(data)
totalrec = len(j)
- else:
- j = data
except BaseException as e:
- self.edprint("Unable to parse data from server: " + str(e))
+ self.edprint(f"Unable to parse data from server: {e}")
return
- if not raw:
+ if raw:
+ out = data
+ else:
if self.ownopts['cli.output'] == "json":
out = json.dumps(j, indent=4, separators=(',', ': '))
@@ -468,10 +473,8 @@ class SpiderFootCli(cmd.Cmd):
out = self.pretty(j, titlemap=titles)
if not out:
- self.edprint("Unknown output format '" + self.ownopts['cli.output'] + "'.")
+ self.edprint(f"Unknown output format '{self.ownopts['cli.output']}'.")
return
- else:
- out = data
c = self.myparseline(cmd)
@@ -479,7 +482,7 @@ class SpiderFootCli(cmd.Cmd):
if len(c[1]) == 0:
self.dprint(out, plain=True)
if total:
- self.dprint("Total records: " + str(totalrec))
+ self.dprint(f"Total records: {totalrec}")
return
for pc in c[1]:
@@ -524,9 +527,9 @@ class SpiderFootCli(cmd.Cmd):
f.write(out)
f.close()
except BaseException as e:
- self.edprint("Unable to write to file: " + str(e))
+ self.edprint(f"Unable to write to file: {e}")
return
- self.dprint("Successfully wrote to file '" + str(pipeargs) + "'.")
+ self.dprint(f"Successfully wrote to file '{pipeargs}'.")
return
out = newout
@@ -629,7 +632,7 @@ class SpiderFootCli(cmd.Cmd):
return
sid = c[0][0]
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanopts?id=" + sid)
+ d = self.request(self.ownopts['cli.server_baseurl'] + f"/scanopts?id={sid}")
if not d:
return
j = json.loads(d)
@@ -637,19 +640,20 @@ class SpiderFootCli(cmd.Cmd):
self.dprint("No such scan exists.")
return
- out = "Name: " + j['meta'][0] + "\n"
- out += "ID: " + sid + "\n"
- out += "Target: " + j['meta'][1] + "\n"
- out += "Started: " + j['meta'][3] + "\n"
- out += "Completed: " + j['meta'][4] + "\n"
- out += "Status: " + j['meta'][5] + "\n"
+ out = list()
+ out.append(f"Name: {j['meta'][0]}")
+ out.append(f"ID: {sid}")
+ out.append(f"Target: {j['meta'][1]}")
+ out.append(f"Started: {j['meta'][3]}")
+ out.append(f"Completed: {j['meta'][4]}")
+ out.append(f"Status: {j['meta'][5]}")
if "-c" in c[0]:
- out += "Configuration:\n"
+ out.append("Configuration:")
for k in sorted(j['config']):
- out += " " + k + " = " + j['config'][k] + "\n"
+ out.append(f" {k} = {j['config'][k]}")
- self.send_output(out, line, total=False, raw=True)
+ self.send_output("\n".join(out), line, total=False, raw=True)
# List scans.
def do_scans(self, line):
@@ -705,8 +709,15 @@ class SpiderFootCli(cmd.Cmd):
if "-u" in c[0]:
url = self.ownopts['cli.server_baseurl'] + "/scaneventresultsunique"
+ titles = {
+ "0": "Data"
+ }
else:
url = self.ownopts['cli.server_baseurl'] + "/scaneventresults"
+ titles = {
+ "10": "Type",
+ "1": "Data"
+ }
d = self.request(url, post=post)
if not d:
@@ -716,19 +727,11 @@ class SpiderFootCli(cmd.Cmd):
self.dprint("No results.")
return
- if "-u" in c[0]:
- titles = {
- "0": "Data"
- }
- else:
- titles = {
- "10": "Type",
- "1": "Data"
- }
if "-x" in c[0]:
titles["0"] = "Last Seen"
titles["3"] = "Module"
titles["2"] = "Source Data"
+
d = d.replace("</SFURL>", "").replace("<SFURL>", "")
self.send_output(d, line, titles=titles)
@@ -736,21 +739,16 @@ class SpiderFootCli(cmd.Cmd):
def do_export(self, line):
"""export <sid> [-t type]
Export the scan data for scan ID <sid> as type [type].
- Valid types: csv, json (default: json)."""
+ Valid types: csv, json, gexf (default: json)."""
c = self.myparseline(line)
if len(c[0]) < 1:
self.edprint("Invalid syntax.")
return
+ export_format = 'json'
if '-t' in c[0]:
export_format = c[0][c[0].index("-t") + 1]
- else:
- export_format = 'json'
-
- if export_format not in ['json', 'csv']:
- print("Invalid export format: %s" % export_format)
- return
base_url = self.ownopts['cli.server_baseurl']
post = {"ids": c[0][0]}
@@ -779,6 +777,18 @@ class SpiderFootCli(cmd.Cmd):
self.send_output(res, line, titles=None, total=False, raw=True)
+ elif export_format == 'gexf':
+ res = self.request(base_url + '/scanvizmulti', post=post)
+
+ if not res:
+ self.dprint("No results.")
+ return
+
+ self.send_output(res, line, titles=None, total=False, raw=True)
+
+ else:
+ self.edprint(f"Invalid export format: {export_format}")
+
# Show logs.
def do_logs(self, line):
"""logs <sid> [-l count] [-w]
@@ -787,17 +797,27 @@ class SpiderFootCli(cmd.Cmd):
If -w is supplied, logs will be streamed to the console until
Ctrl-C is entered."""
c = self.myparseline(line)
+
if len(c[0]) < 1:
self.edprint("Invalid syntax.")
return
+
sid = c[0][0]
limit = None
if "-l" in c[0]:
limit = c[0][c[0].index("-l") + 1]
+ if not limit.isdigit():
+ self.edprint(f"Invalid result count: {limit}")
+ return
+
+ limit = int(limit)
+
if "-w" not in c[0]:
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={'id': sid, 'limit': limit})
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'limit': limit}
+ )
if not d:
return
j = json.loads(d)
@@ -816,52 +836,58 @@ class SpiderFootCli(cmd.Cmd):
}
)
return
- else:
- # Get the rowid of the latest log message
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={'id': sid, 'limit': '1'})
- if not d:
- return
- j = json.loads(d)
- if len(j) < 1:
- self.dprint("No logs (yet?).")
- return
+ # Get the rowid of the latest log message
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'limit': '1'}
+ )
+ if not d:
+ return
+
+ j = json.loads(d)
+ if len(j) < 1:
+ self.dprint("No logs (yet?).")
+ return
- rowid = j[0][4]
+ rowid = j[0][4]
- try:
- if not limit:
- limit = 10
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={'id': sid, 'reverse': '1', 'rowId': rowid - int(limit)})
+ if not limit:
+ limit = 10
+
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'reverse': '1', 'rowId': rowid - limit}
+ )
+ if not d:
+ return
+
+ j = json.loads(d)
+ for r in j:
+ # self.send_output(str(r), line, total=False, raw=True)
+ if r[2] == "ERROR":
+ self.edprint(f"{r[1]}: {r[3]}")
+ else:
+ self.dprint(f"{r[1]}: {r[3]}")
+
+ try:
+ while True:
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/scanlog",
+ post={'id': sid, 'reverse': '1', 'rowId': rowid}
+ )
if not d:
return
-
j = json.loads(d)
for r in j:
- # self.send_output(str(r), line, total=False, raw=True)
if r[2] == "ERROR":
- self.edprint(r[1] + ": " + r[3])
+ self.edprint(f"{r[1]}: {r[3]}")
else:
- self.dprint(r[1] + ": " + r[3])
-
- while True:
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={'id': sid, 'reverse': '1', 'rowId': rowid})
- if not d:
- return
- j = json.loads(d)
- for r in j:
- if r[2] == "ERROR":
- self.edprint(r[1] + ": " + r[3])
- else:
- self.dprint(r[1] + ": " + r[3])
-
- rowid = str(r[4])
- time.sleep(0.5)
- except KeyboardInterrupt:
- return
+ self.dprint(f"{r[1]}: {r[3]}")
+ rowid = str(r[4])
+ time.sleep(0.5)
+ except KeyboardInterrupt:
+ return
# Start a new scan.
def do_start(self, line):
@@ -875,14 +901,15 @@ class SpiderFootCli(cmd.Cmd):
Use -w to watch the logs from the scan. Ctrl-C to abort the
logging (but will not abort the scan).
"""
- mods = ""
- types = ""
- usecase = ""
c = self.myparseline(line)
if len(c[0]) < 3:
self.edprint("Invalid syntax.")
return None
+ mods = ""
+ types = ""
+ usecase = ""
+
if "-m" in c[0]:
mods = c[0][c[0].index("-m") + 1]
@@ -911,20 +938,22 @@ class SpiderFootCli(cmd.Cmd):
"typelist": types,
"usecase": usecase
}
- d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",
- post=post)
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/startscan",
+ post=post
+ )
if not d:
return None
s = json.loads(d)
if s[0] == "SUCCESS":
self.dprint("Successfully initiated scan.")
- self.dprint("Scan ID: " + s[1])
+ self.dprint(f"Scan ID: {s[1]}")
else:
- self.dprint("Unable to start scan: " + str(s[1]))
+ self.dprint(f"Unable to start scan: {s[1]}")
if "-w" in c[0]:
- return self.do_logs(s[1] + " -w")
+ return self.do_logs("{s[1]} -w")
return None
@@ -934,12 +963,12 @@ class SpiderFootCli(cmd.Cmd):
Abort the running scan with scan ID, <sid>."""
c = self.myparseline(line)
try:
- id = c[0][0]
+ scan_id = c[0][0]
except BaseException:
self.edprint("Invalid syntax.")
return
- self.request(self.ownopts['cli.server_baseurl'] + f"/stopscan?id={id}")
+ self.request(self.ownopts['cli.server_baseurl'] + f"/stopscan?id={scan_id}")
self.dprint(f"Successfully requested scan {id} to stop. This could take some minutes to complete.")
# Search for data, alias to find
@@ -975,8 +1004,10 @@ class SpiderFootCli(cmd.Cmd):
if "-x" in c[0]:
titles["2"] = "Source Data"
- d = self.request(self.ownopts['cli.server_baseurl'] + "/search",
- post={'value': val, 'id': sid, 'eventType': etype})
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/search",
+ post={'value': val, 'id': sid, 'eventType': etype}
+ )
if not d:
return
j = json.loads(d)
@@ -1012,7 +1043,7 @@ class SpiderFootCli(cmd.Cmd):
"4": "Unique"
}
- d = self.request(self.ownopts['cli.server_baseurl'] + "/scansummary?id=" + sid + "&by=type")
+ d = self.request(self.ownopts['cli.server_baseurl'] + f"/scansummary?id={sid}&by=type")
if not d:
return
@@ -1033,13 +1064,13 @@ class SpiderFootCli(cmd.Cmd):
Delete a scan with scan ID, <sid>."""
c = self.myparseline(line)
try:
- id = c[0][0]
+ scan_id = c[0][0]
except BaseException:
self.edprint("Invalid syntax.")
return
- self.request(self.ownopts['cli.server_baseurl'] + f"/scandelete?id={id}")
- self.dprint(f"Successfully deleted scan {id}.")
+ self.request(self.ownopts['cli.server_baseurl'] + f"/scandelete?id={scan_id}")
+ self.dprint(f"Successfully deleted scan {scan_id}.")
# Override the default help
def print_topics(self, header, cmds, cmdlen, maxcol):
@@ -1070,8 +1101,12 @@ class SpiderFootCli(cmd.Cmd):
["logs", "View/watch logs from a scan."]
]
- self.send_output(json.dumps(helpmap), "", titles={"0": "Command", "1": "Description"},
- total=False)
+ self.send_output(
+ json.dumps(helpmap),
+ "",
+ titles={"0": "Command", "1": "Description"},
+ total=False
+ )
# Get/Set configuration
def do_set(self, line):
@@ -1093,38 +1128,43 @@ class SpiderFootCli(cmd.Cmd):
return
# Local CLI config
- if val and (cfg in self.ownopts or cfg.startswith('$')):
- if not cfg.startswith('$'):
- if type(self.ownopts[cfg]) == bool:
+ if cfg and val:
+ if cfg.startswith('$'):
+ self.ownopts[cfg] = val
+ self.dprint(f"{cfg} set to {val}")
+ return
+
+ if cfg in self.ownopts:
+ if isinstance(self.ownopts[cfg], bool):
if val.lower() == "false" or val == "0":
val = False
else:
val = True
- self.ownopts[cfg] = val
- self.dprint(f"{cfg} set to {val}")
- return
+ self.ownopts[cfg] = val
+ self.dprint(f"{cfg} set to {val}")
+ return
# Get the server-side config
d = self.request(self.ownopts['cli.server_baseurl'] + "/optsraw")
if not d:
+ self.edprint("Unable to obtain SpiderFoot server-side config.")
return
j = list()
serverconfig = dict()
token = "" # nosec
- if not d:
- self.edprint("Unable to obtain SpiderFoot server-side config.")
- else:
- j = json.loads(d)
- if j[0] == "ERROR":
- self.edprint("Error fetching SpiderFoot server-side config.")
- return
- serverconfig = j[1]['data']
- token = j[1]['token']
+ j = json.loads(d)
+ if j[0] == "ERROR":
+ self.edprint("Error fetching SpiderFoot server-side config.")
+ return
+
+ serverconfig = j[1]['data']
+ token = j[1]['token']
self.ddprint(str(serverconfig))
+ # Printing current config, not setting a value
if not cfg or not val:
ks = list(self.ownopts.keys())
ks.sort()
@@ -1139,18 +1179,19 @@ class SpiderFootCli(cmd.Cmd):
if not cfg:
output.append({'opt': k, 'val': c})
- else:
- if cfg == k:
- self.dprint(k + " = " + c, plain=True)
+ continue
+
+ if cfg == k:
+ self.dprint(f"{k} = {c}", plain=True)
for k in sorted(serverconfig.keys()):
if type(serverconfig[k]) == list:
serverconfig[k] = ','.join(serverconfig[k])
if not cfg:
output.append({'opt': k, 'val': str(serverconfig[k])})
- else:
- if cfg == k:
- self.dprint(k + " = " + str(serverconfig[k]), plain=True)
+ continue
+ if cfg == k:
+ self.dprint(f"{k} = {serverconfig[k]}", plain=True)
if len(output) > 0:
self.send_output(
@@ -1204,7 +1245,7 @@ class SpiderFootCli(cmd.Cmd):
j = json.loads(d)
if j[0] == "ERROR":
- self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))
+ self.edprint(f"Error setting SpiderFoot server-side config: {j[1]}")
return
self.dprint(f"{cfg} set to {val}")