SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfcli: Allow setting local options without a server connection (#1219)
bcoles3b501174daa1
diff --git a/sfcli.py b/sfcli.pyindex af6837d9..456785a4 100755--- a/sfcli.py+++ b/sfcli.py@@ -174,8 +174,8 @@ class SpiderFootCli(cmd.Cmd):if self.ownopts['cli.spool_file']:return self.do_set("cli.spool = " + val)- else:- self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")++ self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")return None@@ -191,10 +191,12 @@ class SpiderFootCli(cmd.Cmd):self.dprint(readline.get_history_item(i), plain=True)i += 1return None+if self.ownopts['cli.history']:val = "0"else:val = "1"+return self.do_set("cli.history = " + val)# Run before all commands to handle history and spooling@@ -410,6 +412,10 @@ class SpiderFootCli(cmd.Cmd):# [[ 'blahblah test' ], [[ 'top', '10' ], [ 'grep', 'foo']]]def myparseline(self, cmdline, replace=True):ret = [list(), list()]++ if not cmdline:+ return ret+s = shlex.split(cmdline)for c in s:if c == '|':@@ -816,12 +822,13 @@ class SpiderFootCli(cmd.Cmd):post={'id': sid, 'limit': '1'})if not d:return+j = json.loads(d)if len(j) < 1:self.dprint("No logs (yet?).")return- else:- rowid = j[0][4]++ rowid = j[0][4]try:if not limit:@@ -1085,10 +1092,24 @@ class SpiderFootCli(cmd.Cmd):self.edprint("Invalid syntax.")return+ # Local CLI config+ if val and (cfg in self.ownopts or cfg.startswith('$')):+ if not cfg.startswith('$'):+ if type(self.ownopts[cfg]) == bool:+ if val.lower() == "false" or val == "0":+ val = False+ else:+ val = True++ self.ownopts[cfg] = val+ self.dprint(f"{cfg} set to {val}")+ return+# Get the server-side configd = self.request(self.ownopts['cli.server_baseurl'] + "/optsraw")if not d:return+j = list()serverconfig = dict()token = "" # nosec@@ -1099,9 +1120,8 @@ class SpiderFootCli(cmd.Cmd):if j[0] == "ERROR":self.edprint("Error fetching SpiderFoot server-side config.")return- else:- serverconfig = j[1]['data']- token = j[1]['token']+ serverconfig = j[1]['data']+ token = j[1]['token']self.ddprint(str(serverconfig))@@ -1142,63 +1162,53 @@ class SpiderFootCli(cmd.Cmd):returnif val:- # Local CLI config- if cfg in self.ownopts or cfg.startswith('$'):- if not cfg.startswith('$'):- if type(self.ownopts[cfg]) == bool:- if val.lower() == "false" or val == "0":- val = False- else:- val = True+ # submit all non-CLI vars to the SF server+ confdata = dict()+ found = False+ for k in serverconfig:+ if k == cfg:+ serverconfig[k] = val+ found = True++ if not found:+ self.edprint("Variable not found, so not set.")+ return- self.ownopts[cfg] = val- self.dprint(cfg + " set to " + str(val))+ # Sanitize the data before sending it to the server+ for k in serverconfig:+ optstr = ":".join(k.split(".")[1:])+ if type(serverconfig[k]) == bool:+ if serverconfig[k]:+ confdata[optstr] = "1"+ else:+ confdata[optstr] = "0"+ if type(serverconfig[k]) == list:+ # If set by the user, it must already be a+ # string, not a list+ confdata[optstr] = ','.join(serverconfig[k])+ if type(serverconfig[k]) == int:+ confdata[optstr] = str(serverconfig[k])+ if type(serverconfig[k]) == str:+ confdata[optstr] = serverconfig[k]++ self.ddprint(str(confdata))+ d = self.request(+ self.ownopts['cli.server_baseurl'] + "/savesettingsraw",+ post={'token': token, 'allopts': json.dumps(confdata)}+ )+ j = list()++ if not d:+ self.edprint("Unable to set SpiderFoot server-side config.")return- # Server-side config- else:- # submit all non-CLI vars to the SF server- confdata = dict()- found = False- for k in serverconfig:- if k == cfg:- serverconfig[k] = val- found = True-- if not found:- self.edprint("Variable not found, so not set.")- return- # Sanitize the data before sending it to the server- for k in serverconfig:- optstr = ":".join(k.split(".")[1:])- if type(serverconfig[k]) == bool:- if not serverconfig[k]:- confdata[optstr] = "0"- else:- confdata[optstr] = "1"- if type(serverconfig[k]) == list:- # If set by the user, it must already be a- # string, not a list- confdata[optstr] = ','.join(serverconfig[k])- if type(serverconfig[k]) == int:- confdata[optstr] = str(serverconfig[k])- if type(serverconfig[k]) == str:- confdata[optstr] = serverconfig[k]-- self.ddprint(str(confdata))- d = self.request(self.ownopts['cli.server_baseurl'] + "/savesettingsraw",- post={'token': token, 'allopts': json.dumps(confdata)})- j = list()- if not d:- self.edprint("Unable to set SpiderFoot server-side config.")- return- else:- j = json.loads(d)- if j[0] == "ERROR":- self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))- return- self.dprint(cfg + " set to " + str(val))- return+ j = json.loads(d)+ if j[0] == "ERROR":+ self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))+ return++ self.dprint(f"{cfg} set to {val}")+ returnif cfg not in self.ownopts:self.edprint("Variable not found, so not set. Did you mean to use a $ variable?")diff --git a/test/unit/test_spiderfootcli.py b/test/unit/test_spiderfootcli.pyindex a34295e2..1958856b 100644--- a/test/unit/test_spiderfootcli.py+++ b/test/unit/test_spiderfootcli.py@@ -100,35 +100,19 @@ class TestSpiderFootCli(unittest.TestCase):self.assertIsInstance(default, list)self.assertEqual([], default)- def test_dprint(self):- """- Test dprint(self, msg, err=False, deb=False, plain=False, color=None)- """- sfcli = SpiderFootCli()-- io_output = io.StringIO()- sys.stdout = io_output- sfcli.dprint("example output")- sys.stdout = sys.__stdout__- output = io_output.getvalue()-- self.assertIn("example output", output)-- @unittest.skip("todo")def test_do_debug_should_toggle_debug(self):"""Test do_debug(self, line)"""sfcli = SpiderFootCli(self.cli_default_options)- sfcli.do_debug(0)+ sfcli.do_debug(None)initial_debug_state = sfcli.ownopts['cli.debug']- sfcli.do_debug(1)+ sfcli.do_debug(None)new_debug_state = sfcli.ownopts['cli.debug']self.assertNotEqual(initial_debug_state, new_debug_state)- @unittest.skip("todo")def test_do_spool_should_toggle_spool(self):"""Test do_spool(self, line)@@ -137,30 +121,33 @@ class TestSpiderFootCli(unittest.TestCase):sfcli.ownopts['cli.spool_file'] = '/dev/null'- sfcli.do_spool(0)+ sfcli.do_spool(None)initial_spool_state = sfcli.ownopts['cli.spool']- sfcli.do_spool(1)+ sfcli.do_spool(None)new_spool_state = sfcli.ownopts['cli.spool']self.assertNotEqual(initial_spool_state, new_spool_state)- @unittest.skip("todo")- def test_do_history(self):+ def test_do_history_should_toggle_history_option(self):"""Test do_history(self, line)"""sfcli = SpiderFootCli(self.cli_default_options)- sfcli.do_history(None)+ sfcli.do_history("0")+ initial_history_state = sfcli.ownopts['cli.history']+ sfcli.do_history("1")+ new_history_state = sfcli.ownopts['cli.history']- self.assertEqual('TBD', 'TBD')+ self.assertNotEqual(initial_history_state, new_history_state)- @unittest.skip("todo")- def test_precmd_should_print_line_to_history_file(self):+ def test_precmd_should_return_line(self):"""Test precmd(self, line)"""sfcli = SpiderFootCli()+ sfcli.ownopts['cli.history'] = False+ sfcli.ownopts['cli.spool'] = Falseline = "example line"@@ -168,14 +155,14 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual(line, precmd)- self.assertEqual('TBD', 'TBD')-@unittest.skip("todo")- def test_precmd_should_print_line_to_spool_file(self):+ def test_precmd_should_print_line_to_history_file(self):"""Test precmd(self, line)"""sfcli = SpiderFootCli()+ sfcli.ownopts['cli.history'] = True+ sfcli.ownopts['cli.spool'] = Falseline = "example line"@@ -186,11 +173,14 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')@unittest.skip("todo")- def test_precmd_should_return_line(self):+ def test_precmd_should_print_line_to_spool_file(self):"""Test precmd(self, line)"""sfcli = SpiderFootCli()+ sfcli.ownopts['cli.history'] = False+ sfcli.ownopts['cli.spool'] = True+ sfcli.ownopts['cli.spool_file'] = '/dev/null'line = "example line"@@ -198,12 +188,47 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual(line, precmd)- @unittest.skip("todo")- def test_ddprint(self):+ self.assertEqual('TBD', 'TBD')++ def test_dprint_should_print_if_debug_option_is_set(self):+ """+ Test dprint(self, msg, err=False, deb=False, plain=False, color=None)+ """+ sfcli = SpiderFootCli()+ sfcli.ownopts['cli.debug'] = True+ sfcli.ownopts['cli.spool'] = False++ io_output = io.StringIO()+ sys.stdout = io_output+ sfcli.dprint("example output")+ sys.stdout = sys.__stdout__+ output = io_output.getvalue()++ self.assertIn("example output", output)++ def test_dprint_should_not_print_unless_debug_option_is_set(self):+ """+ Test dprint(self, msg, err=False, deb=False, plain=False, color=None)+ """+ sfcli = SpiderFootCli()+ sfcli.ownopts['cli.debug'] = False+ sfcli.ownopts['cli.spool'] = False++ io_output = io.StringIO()+ sys.stdout = io_output+ sfcli.dprint("example output")+ sys.stdout = sys.__stdout__+ output = io_output.getvalue()++ self.assertIn("", output)++ def test_ddprint_should_print_if_debug_option_is_set(self):"""Test ddprint(self, msg)"""sfcli = SpiderFootCli()+ sfcli.ownopts['cli.debug'] = True+ sfcli.ownopts['cli.spool'] = Falseio_output = io.StringIO()sys.stdout = io_output@@ -213,13 +238,29 @@ class TestSpiderFootCli(unittest.TestCase):self.assertIn("example debug output", output)- self.assertEqual('TBD', 'TBD')+ def test_ddprint_should_not_print_unless_debug_option_is_set(self):+ """+ Test ddprint(self, msg)+ """+ sfcli = SpiderFootCli()+ sfcli.ownopts['cli.debug'] = False+ sfcli.ownopts['cli.spool'] = False- def test_edprint(self):+ io_output = io.StringIO()+ sys.stdout = io_output+ sfcli.ddprint("example debug output")+ sys.stdout = sys.__stdout__+ output = io_output.getvalue()++ self.assertEqual("", output)++ def test_edprint_should_print_error_regardless_of_debug_option(self):"""Test edprint(self, msg)"""sfcli = SpiderFootCli()+ sfcli.ownopts['cli.debug'] = False+ sfcli.ownopts['cli.spool'] = Falseio_output = io.StringIO()sys.stdout = io_output@@ -241,16 +282,6 @@ class TestSpiderFootCli(unittest.TestCase):pretty = sfcli.pretty(invalid_type)self.assertEqual("", pretty)- @unittest.skip("todo")- def test_request(self):- """- Test request(self, url, post=None)- """- sfcli = SpiderFootCli()- sfcli.request(None, None)-- self.assertEqual('TBD', 'TBD')-def test_request_invalid_url_should_return_none(self):"""Test request(self, url, post=None)@@ -285,6 +316,13 @@ class TestSpiderFootCli(unittest.TestCase):Test myparseline(self, cmdline, replace=True)"""sfcli = SpiderFootCli()+ parsed_line = sfcli.myparseline(None)++ self.assertEqual(len(parsed_line), 2)+ self.assertIsInstance(parsed_line, list)+ self.assertIsInstance(parsed_line[0], list)+ self.assertIsInstance(parsed_line[1], list)+parsed_line = sfcli.myparseline("")self.assertEqual(len(parsed_line), 2)@@ -308,7 +346,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_query(self):"""Test do_query(self, line)@@ -354,7 +391,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_scaninfo(self):"""Test do_scaninfo(self, line)@@ -373,7 +409,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_data(self):"""Test do_data(self, line)@@ -383,7 +418,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_export(self):"""Test do_export(self, line)@@ -393,7 +427,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_logs(self):"""Test do_logs(self, line)@@ -403,7 +436,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_start(self):"""Test do_start(self, line)@@ -413,7 +445,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_stop(self):"""Test do_stop(self, line)@@ -423,7 +454,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_search(self):"""Test do_search(self, line)@@ -433,7 +463,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_find(self):"""Test do_find(self, line)@@ -443,7 +472,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_summary(self):"""Test do_summary(self, line)@@ -453,7 +481,6 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")def test_do_delete(self):"""Test do_delete(self, line)@@ -480,15 +507,17 @@ class TestSpiderFootCli(unittest.TestCase):self.assertEqual('TBD', 'TBD')- @unittest.skip("todo")- def test_do_set(self):+ def test_do_set_should_set_option(self):"""Test do_set(self, line)"""sfcli = SpiderFootCli()- sfcli.do_set(None)+ sfcli.ownopts['cli.test_opt'] = None- self.assertEqual('TBD', 'TBD')+ sfcli.do_set('cli.test_opt = "test value"')+ new_test_opt = sfcli.ownopts['cli.test_opt']++ self.assertEqual(new_test_opt, 'test value')def test_do_shell(self):"""