SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfcli: Cleanup (#1302)
bcolesb7eecdc5bc4e
diff --git a/sfcli.py b/sfcli.pyindex e9677781..200d7bb1 100755--- a/sfcli.py+++ b/sfcli.py@@ -355,7 +355,7 @@ class SpiderFootCli(cmd.Cmd):return Noneif not isinstance(url, str):- self.edprint("Invalid request URL: %s" % url)+ self.edprint(f"Invalid request URL: {url}")return None# logging.basicConfig()@@ -363,14 +363,13 @@ class SpiderFootCli(cmd.Cmd):# requests_log = logging.getLogger("requests.packages.urllib3")# requests_log.setLevel(logging.DEBUG)# requests_log.propagate = True- try:- headers = {- "User-agent": "SpiderFoot-CLI/" + self.version,- "Accept": "application/json"- }+ headers = {+ "User-agent": "SpiderFoot-CLI/" + self.version,+ "Accept": "application/json"+ }- self.ddprint("Fetching: " + url)- self.ddprint("Posting: " + str(post))+ try:+ self.ddprint(f"Fetching: {url}")if not post:r = requests.get(url,@@ -382,6 +381,7 @@ class SpiderFootCli(cmd.Cmd):))else:+ self.ddprint(f"Posting: {post}")r = requests.post(url,headers=headers,@@ -392,13 +392,12 @@ class SpiderFootCli(cmd.Cmd):),data=post)- self.ddprint("Response: " + str(r))+ self.ddprint(f"Response: {r}")if r.status_code == requests.codes.ok: # pylint: disable=no-memberreturn r.text- else:- r.raise_for_status()+ r.raise_for_status()except BaseException as e:- self.edprint("Failed communicating with server: " + str(e))+ self.edprint(f"Failed communicating with server: {e}")return Nonedef emptyline(self):@@ -416,7 +415,12 @@ class SpiderFootCli(cmd.Cmd):if not cmdline:return ret- s = shlex.split(cmdline)+ try:+ s = shlex.split(cmdline)+ except Exception as e:+ self.edprint(f"Error parsing command: {e}")+ return ret+for c in s:if c == '|':break@@ -436,9 +440,8 @@ class SpiderFootCli(cmd.Cmd):if t == '|':i += 1ret[1].append(list())- continue# Replace variables- if t.startswith("$") and t in self.ownopts:+ elif t.startswith("$") and t in self.ownopts:ret[1][i].append(self.ownopts[t])else:ret[1][i].append(t)@@ -448,19 +451,21 @@ class SpiderFootCli(cmd.Cmd):# Send the command output to the user, processing the pipes# that may have been used.def send_output(self, data, cmd, titles=None, total=True, raw=False):- totalrec = 0out = Nonetry:- if not raw:+ if raw:+ j = data+ totalrec = 0+ else:j = json.loads(data)totalrec = len(j)- else:- j = dataexcept BaseException as e:- self.edprint("Unable to parse data from server: " + str(e))+ self.edprint(f"Unable to parse data from server: {e}")return- if not raw:+ if raw:+ out = data+ else:if self.ownopts['cli.output'] == "json":out = json.dumps(j, indent=4, separators=(',', ': '))@@ -468,10 +473,8 @@ class SpiderFootCli(cmd.Cmd):out = self.pretty(j, titlemap=titles)if not out:- self.edprint("Unknown output format '" + self.ownopts['cli.output'] + "'.")+ self.edprint(f"Unknown output format '{self.ownopts['cli.output']}'.")return- else:- out = datac = self.myparseline(cmd)@@ -479,7 +482,7 @@ class SpiderFootCli(cmd.Cmd):if len(c[1]) == 0:self.dprint(out, plain=True)if total:- self.dprint("Total records: " + str(totalrec))+ self.dprint(f"Total records: {totalrec}")returnfor pc in c[1]:@@ -524,9 +527,9 @@ class SpiderFootCli(cmd.Cmd):f.write(out)f.close()except BaseException as e:- self.edprint("Unable to write to file: " + str(e))+ self.edprint(f"Unable to write to file: {e}")return- self.dprint("Successfully wrote to file '" + str(pipeargs) + "'.")+ self.dprint(f"Successfully wrote to file '{pipeargs}'.")returnout = newout@@ -629,7 +632,7 @@ class SpiderFootCli(cmd.Cmd):returnsid = c[0][0]- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanopts?id=" + sid)+ d = self.request(self.ownopts['cli.server_baseurl'] + f"/scanopts?id={sid}")if not d:returnj = json.loads(d)@@ -637,19 +640,20 @@ class SpiderFootCli(cmd.Cmd):self.dprint("No such scan exists.")return- out = "Name: " + j['meta'][0] + "\n"- out += "ID: " + sid + "\n"- out += "Target: " + j['meta'][1] + "\n"- out += "Started: " + j['meta'][3] + "\n"- out += "Completed: " + j['meta'][4] + "\n"- out += "Status: " + j['meta'][5] + "\n"+ out = list()+ out.append(f"Name: {j['meta'][0]}")+ out.append(f"ID: {sid}")+ out.append(f"Target: {j['meta'][1]}")+ out.append(f"Started: {j['meta'][3]}")+ out.append(f"Completed: {j['meta'][4]}")+ out.append(f"Status: {j['meta'][5]}")if "-c" in c[0]:- out += "Configuration:\n"+ out.append("Configuration:")for k in sorted(j['config']):- out += " " + k + " = " + j['config'][k] + "\n"+ out.append(f" {k} = {j['config'][k]}")- self.send_output(out, line, total=False, raw=True)+ self.send_output("\n".join(out), line, total=False, raw=True)# List scans.def do_scans(self, line):@@ -705,8 +709,15 @@ class SpiderFootCli(cmd.Cmd):if "-u" in c[0]:url = self.ownopts['cli.server_baseurl'] + "/scaneventresultsunique"+ titles = {+ "0": "Data"+ }else:url = self.ownopts['cli.server_baseurl'] + "/scaneventresults"+ titles = {+ "10": "Type",+ "1": "Data"+ }d = self.request(url, post=post)if not d:@@ -716,19 +727,11 @@ class SpiderFootCli(cmd.Cmd):self.dprint("No results.")return- if "-u" in c[0]:- titles = {- "0": "Data"- }- else:- titles = {- "10": "Type",- "1": "Data"- }if "-x" in c[0]:titles["0"] = "Last Seen"titles["3"] = "Module"titles["2"] = "Source Data"+d = d.replace("</SFURL>", "").replace("<SFURL>", "")self.send_output(d, line, titles=titles)@@ -736,21 +739,16 @@ class SpiderFootCli(cmd.Cmd):def do_export(self, line):"""export <sid> [-t type]Export the scan data for scan ID <sid> as type [type].- Valid types: csv, json (default: json)."""+ Valid types: csv, json, gexf (default: json)."""c = self.myparseline(line)if len(c[0]) < 1:self.edprint("Invalid syntax.")return+ export_format = 'json'if '-t' in c[0]:export_format = c[0][c[0].index("-t") + 1]- else:- export_format = 'json'-- if export_format not in ['json', 'csv']:- print("Invalid export format: %s" % export_format)- returnbase_url = self.ownopts['cli.server_baseurl']post = {"ids": c[0][0]}@@ -779,6 +777,18 @@ class SpiderFootCli(cmd.Cmd):self.send_output(res, line, titles=None, total=False, raw=True)+ elif export_format == 'gexf':+ res = self.request(base_url + '/scanvizmulti', post=post)++ if not res:+ self.dprint("No results.")+ return++ self.send_output(res, line, titles=None, total=False, raw=True)++ else:+ self.edprint(f"Invalid export format: {export_format}")+# Show logs.def do_logs(self, line):"""logs <sid> [-l count] [-w]@@ -787,17 +797,27 @@ class SpiderFootCli(cmd.Cmd):If -w is supplied, logs will be streamed to the console untilCtrl-C is entered."""c = self.myparseline(line)+if len(c[0]) < 1:self.edprint("Invalid syntax.")return+sid = c[0][0]limit = Noneif "-l" in c[0]:limit = c[0][c[0].index("-l") + 1]+ if not limit.isdigit():+ self.edprint(f"Invalid result count: {limit}")+ return++ limit = int(limit)+if "-w" not in c[0]:- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={'id': sid, 'limit': limit})+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/scanlog",+ post={'id': sid, 'limit': limit}+ )if not d:returnj = json.loads(d)@@ -816,52 +836,58 @@ class SpiderFootCli(cmd.Cmd):})return- else:- # Get the rowid of the latest log message- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={'id': sid, 'limit': '1'})- if not d:- return- j = json.loads(d)- if len(j) < 1:- self.dprint("No logs (yet?).")- return+ # Get the rowid of the latest log message+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/scanlog",+ post={'id': sid, 'limit': '1'}+ )+ if not d:+ return++ j = json.loads(d)+ if len(j) < 1:+ self.dprint("No logs (yet?).")+ return- rowid = j[0][4]+ rowid = j[0][4]- try:- if not limit:- limit = 10- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={'id': sid, 'reverse': '1', 'rowId': rowid - int(limit)})+ if not limit:+ limit = 10++ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/scanlog",+ post={'id': sid, 'reverse': '1', 'rowId': rowid - limit}+ )+ if not d:+ return++ j = json.loads(d)+ for r in j:+ # self.send_output(str(r), line, total=False, raw=True)+ if r[2] == "ERROR":+ self.edprint(f"{r[1]}: {r[3]}")+ else:+ self.dprint(f"{r[1]}: {r[3]}")++ try:+ while True:+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/scanlog",+ post={'id': sid, 'reverse': '1', 'rowId': rowid}+ )if not d:return-j = json.loads(d)for r in j:- # self.send_output(str(r), line, total=False, raw=True)if r[2] == "ERROR":- self.edprint(r[1] + ": " + r[3])+ self.edprint(f"{r[1]}: {r[3]}")else:- self.dprint(r[1] + ": " + r[3])-- while True:- d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={'id': sid, 'reverse': '1', 'rowId': rowid})- if not d:- return- j = json.loads(d)- for r in j:- if r[2] == "ERROR":- self.edprint(r[1] + ": " + r[3])- else:- self.dprint(r[1] + ": " + r[3])-- rowid = str(r[4])- time.sleep(0.5)- except KeyboardInterrupt:- return+ self.dprint(f"{r[1]}: {r[3]}")+ rowid = str(r[4])+ time.sleep(0.5)+ except KeyboardInterrupt:+ return# Start a new scan.def do_start(self, line):@@ -875,14 +901,15 @@ class SpiderFootCli(cmd.Cmd):Use -w to watch the logs from the scan. Ctrl-C to abort thelogging (but will not abort the scan)."""- mods = ""- types = ""- usecase = ""c = self.myparseline(line)if len(c[0]) < 3:self.edprint("Invalid syntax.")return None+ mods = ""+ types = ""+ usecase = ""+if "-m" in c[0]:mods = c[0][c[0].index("-m") + 1]@@ -911,20 +938,22 @@ class SpiderFootCli(cmd.Cmd):"typelist": types,"usecase": usecase}- d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",- post=post)+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/startscan",+ post=post+ )if not d:return Nones = json.loads(d)if s[0] == "SUCCESS":self.dprint("Successfully initiated scan.")- self.dprint("Scan ID: " + s[1])+ self.dprint(f"Scan ID: {s[1]}")else:- self.dprint("Unable to start scan: " + str(s[1]))+ self.dprint(f"Unable to start scan: {s[1]}")if "-w" in c[0]:- return self.do_logs(s[1] + " -w")+ return self.do_logs("{s[1]} -w")return None@@ -934,12 +963,12 @@ class SpiderFootCli(cmd.Cmd):Abort the running scan with scan ID, <sid>."""c = self.myparseline(line)try:- id = c[0][0]+ scan_id = c[0][0]except BaseException:self.edprint("Invalid syntax.")return- self.request(self.ownopts['cli.server_baseurl'] + f"/stopscan?id={id}")+ self.request(self.ownopts['cli.server_baseurl'] + f"/stopscan?id={scan_id}")self.dprint(f"Successfully requested scan {id} to stop. This could take some minutes to complete.")# Search for data, alias to find@@ -975,8 +1004,10 @@ class SpiderFootCli(cmd.Cmd):if "-x" in c[0]:titles["2"] = "Source Data"- d = self.request(self.ownopts['cli.server_baseurl'] + "/search",- post={'value': val, 'id': sid, 'eventType': etype})+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/search",+ post={'value': val, 'id': sid, 'eventType': etype}+ )if not d:returnj = json.loads(d)@@ -1012,7 +1043,7 @@ class SpiderFootCli(cmd.Cmd):"4": "Unique"}- d = self.request(self.ownopts['cli.server_baseurl'] + "/scansummary?id=" + sid + "&by=type")+ d = self.request(self.ownopts['cli.server_baseurl'] + f"/scansummary?id={sid}&by=type")if not d:return@@ -1033,13 +1064,13 @@ class SpiderFootCli(cmd.Cmd):Delete a scan with scan ID, <sid>."""c = self.myparseline(line)try:- id = c[0][0]+ scan_id = c[0][0]except BaseException:self.edprint("Invalid syntax.")return- self.request(self.ownopts['cli.server_baseurl'] + f"/scandelete?id={id}")- self.dprint(f"Successfully deleted scan {id}.")+ self.request(self.ownopts['cli.server_baseurl'] + f"/scandelete?id={scan_id}")+ self.dprint(f"Successfully deleted scan {scan_id}.")# Override the default helpdef print_topics(self, header, cmds, cmdlen, maxcol):@@ -1070,8 +1101,12 @@ class SpiderFootCli(cmd.Cmd):["logs", "View/watch logs from a scan."]]- self.send_output(json.dumps(helpmap), "", titles={"0": "Command", "1": "Description"},- total=False)+ self.send_output(+ json.dumps(helpmap),+ "",+ titles={"0": "Command", "1": "Description"},+ total=False+ )# Get/Set configurationdef do_set(self, line):@@ -1093,38 +1128,43 @@ class SpiderFootCli(cmd.Cmd):return# Local CLI config- if val and (cfg in self.ownopts or cfg.startswith('$')):- if not cfg.startswith('$'):- if type(self.ownopts[cfg]) == bool:+ if cfg and val:+ if cfg.startswith('$'):+ self.ownopts[cfg] = val+ self.dprint(f"{cfg} set to {val}")+ return++ if cfg in self.ownopts:+ if isinstance(self.ownopts[cfg], bool):if val.lower() == "false" or val == "0":val = Falseelse:val = True- self.ownopts[cfg] = val- self.dprint(f"{cfg} set to {val}")- return+ self.ownopts[cfg] = val+ self.dprint(f"{cfg} set to {val}")+ return# Get the server-side configd = self.request(self.ownopts['cli.server_baseurl'] + "/optsraw")if not d:+ self.edprint("Unable to obtain SpiderFoot server-side config.")returnj = list()serverconfig = dict()token = "" # nosec- if not d:- self.edprint("Unable to obtain SpiderFoot server-side config.")- else:- j = json.loads(d)- if j[0] == "ERROR":- self.edprint("Error fetching SpiderFoot server-side config.")- return- serverconfig = j[1]['data']- token = j[1]['token']+ j = json.loads(d)+ if j[0] == "ERROR":+ self.edprint("Error fetching SpiderFoot server-side config.")+ return++ serverconfig = j[1]['data']+ token = j[1]['token']self.ddprint(str(serverconfig))+ # Printing current config, not setting a valueif not cfg or not val:ks = list(self.ownopts.keys())ks.sort()@@ -1139,18 +1179,19 @@ class SpiderFootCli(cmd.Cmd):if not cfg:output.append({'opt': k, 'val': c})- else:- if cfg == k:- self.dprint(k + " = " + c, plain=True)+ continue++ if cfg == k:+ self.dprint(f"{k} = {c}", plain=True)for k in sorted(serverconfig.keys()):if type(serverconfig[k]) == list:serverconfig[k] = ','.join(serverconfig[k])if not cfg:output.append({'opt': k, 'val': str(serverconfig[k])})- else:- if cfg == k:- self.dprint(k + " = " + str(serverconfig[k]), plain=True)+ continue+ if cfg == k:+ self.dprint(f"{k} = {serverconfig[k]}", plain=True)if len(output) > 0:self.send_output(@@ -1204,7 +1245,7 @@ class SpiderFootCli(cmd.Cmd):j = json.loads(d)if j[0] == "ERROR":- self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))+ self.edprint(f"Error setting SpiderFoot server-side config: {j[1]}")returnself.dprint(f"{cfg} set to {val}")