Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfcli: Allow setting local options without a server connection (#1219)

bcoles3b501174daa1
diff --git a/sfcli.py b/sfcli.py
index af6837d9..456785a4 100755
--- a/sfcli.py
+++ b/sfcli.py
@@ -174,8 +174,8 @@ class SpiderFootCli(cmd.Cmd):
if self.ownopts['cli.spool_file']:
return self.do_set("cli.spool = " + val)
- else:
- self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")
+
+ self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")
return None
@@ -191,10 +191,12 @@ class SpiderFootCli(cmd.Cmd):
self.dprint(readline.get_history_item(i), plain=True)
i += 1
return None
+
if self.ownopts['cli.history']:
val = "0"
else:
val = "1"
+
return self.do_set("cli.history = " + val)
# Run before all commands to handle history and spooling
@@ -410,6 +412,10 @@ class SpiderFootCli(cmd.Cmd):
# [[ 'blahblah test' ], [[ 'top', '10' ], [ 'grep', 'foo']]]
def myparseline(self, cmdline, replace=True):
ret = [list(), list()]
+
+ if not cmdline:
+ return ret
+
s = shlex.split(cmdline)
for c in s:
if c == '|':
@@ -816,12 +822,13 @@ class SpiderFootCli(cmd.Cmd):
post={'id': sid, 'limit': '1'})
if not d:
return
+
j = json.loads(d)
if len(j) < 1:
self.dprint("No logs (yet?).")
return
- else:
- rowid = j[0][4]
+
+ rowid = j[0][4]
try:
if not limit:
@@ -1085,10 +1092,24 @@ class SpiderFootCli(cmd.Cmd):
self.edprint("Invalid syntax.")
return
+ # Local CLI config
+ if val and (cfg in self.ownopts or cfg.startswith('$')):
+ if not cfg.startswith('$'):
+ if type(self.ownopts[cfg]) == bool:
+ if val.lower() == "false" or val == "0":
+ val = False
+ else:
+ val = True
+
+ self.ownopts[cfg] = val
+ self.dprint(f"{cfg} set to {val}")
+ return
+
# Get the server-side config
d = self.request(self.ownopts['cli.server_baseurl'] + "/optsraw")
if not d:
return
+
j = list()
serverconfig = dict()
token = "" # nosec
@@ -1099,9 +1120,8 @@ class SpiderFootCli(cmd.Cmd):
if j[0] == "ERROR":
self.edprint("Error fetching SpiderFoot server-side config.")
return
- else:
- serverconfig = j[1]['data']
- token = j[1]['token']
+ serverconfig = j[1]['data']
+ token = j[1]['token']
self.ddprint(str(serverconfig))
@@ -1142,63 +1162,53 @@ class SpiderFootCli(cmd.Cmd):
return
if val:
- # Local CLI config
- if cfg in self.ownopts or cfg.startswith('$'):
- if not cfg.startswith('$'):
- if type(self.ownopts[cfg]) == bool:
- if val.lower() == "false" or val == "0":
- val = False
- else:
- val = True
+ # submit all non-CLI vars to the SF server
+ confdata = dict()
+ found = False
+ for k in serverconfig:
+ if k == cfg:
+ serverconfig[k] = val
+ found = True
+
+ if not found:
+ self.edprint("Variable not found, so not set.")
+ return
- self.ownopts[cfg] = val
- self.dprint(cfg + " set to " + str(val))
+ # Sanitize the data before sending it to the server
+ for k in serverconfig:
+ optstr = ":".join(k.split(".")[1:])
+ if type(serverconfig[k]) == bool:
+ if serverconfig[k]:
+ confdata[optstr] = "1"
+ else:
+ confdata[optstr] = "0"
+ if type(serverconfig[k]) == list:
+ # If set by the user, it must already be a
+ # string, not a list
+ confdata[optstr] = ','.join(serverconfig[k])
+ if type(serverconfig[k]) == int:
+ confdata[optstr] = str(serverconfig[k])
+ if type(serverconfig[k]) == str:
+ confdata[optstr] = serverconfig[k]
+
+ self.ddprint(str(confdata))
+ d = self.request(
+ self.ownopts['cli.server_baseurl'] + "/savesettingsraw",
+ post={'token': token, 'allopts': json.dumps(confdata)}
+ )
+ j = list()
+
+ if not d:
+ self.edprint("Unable to set SpiderFoot server-side config.")
return
- # Server-side config
- else:
- # submit all non-CLI vars to the SF server
- confdata = dict()
- found = False
- for k in serverconfig:
- if k == cfg:
- serverconfig[k] = val
- found = True
-
- if not found:
- self.edprint("Variable not found, so not set.")
- return
- # Sanitize the data before sending it to the server
- for k in serverconfig:
- optstr = ":".join(k.split(".")[1:])
- if type(serverconfig[k]) == bool:
- if not serverconfig[k]:
- confdata[optstr] = "0"
- else:
- confdata[optstr] = "1"
- if type(serverconfig[k]) == list:
- # If set by the user, it must already be a
- # string, not a list
- confdata[optstr] = ','.join(serverconfig[k])
- if type(serverconfig[k]) == int:
- confdata[optstr] = str(serverconfig[k])
- if type(serverconfig[k]) == str:
- confdata[optstr] = serverconfig[k]
-
- self.ddprint(str(confdata))
- d = self.request(self.ownopts['cli.server_baseurl'] + "/savesettingsraw",
- post={'token': token, 'allopts': json.dumps(confdata)})
- j = list()
- if not d:
- self.edprint("Unable to set SpiderFoot server-side config.")
- return
- else:
- j = json.loads(d)
- if j[0] == "ERROR":
- self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))
- return
- self.dprint(cfg + " set to " + str(val))
- return
+ j = json.loads(d)
+ if j[0] == "ERROR":
+ self.edprint("Error setting SpiderFoot server-side config: " + str(j[1]))
+ return
+
+ self.dprint(f"{cfg} set to {val}")
+ return
if cfg not in self.ownopts:
self.edprint("Variable not found, so not set. Did you mean to use a $ variable?")
diff --git a/test/unit/test_spiderfootcli.py b/test/unit/test_spiderfootcli.py
index a34295e2..1958856b 100644
--- a/test/unit/test_spiderfootcli.py
+++ b/test/unit/test_spiderfootcli.py
@@ -100,35 +100,19 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertIsInstance(default, list)
self.assertEqual([], default)
- def test_dprint(self):
- """
- Test dprint(self, msg, err=False, deb=False, plain=False, color=None)
- """
- sfcli = SpiderFootCli()
-
- io_output = io.StringIO()
- sys.stdout = io_output
- sfcli.dprint("example output")
- sys.stdout = sys.__stdout__
- output = io_output.getvalue()
-
- self.assertIn("example output", output)
-
- @unittest.skip("todo")
def test_do_debug_should_toggle_debug(self):
"""
Test do_debug(self, line)
"""
sfcli = SpiderFootCli(self.cli_default_options)
- sfcli.do_debug(0)
+ sfcli.do_debug(None)
initial_debug_state = sfcli.ownopts['cli.debug']
- sfcli.do_debug(1)
+ sfcli.do_debug(None)
new_debug_state = sfcli.ownopts['cli.debug']
self.assertNotEqual(initial_debug_state, new_debug_state)
- @unittest.skip("todo")
def test_do_spool_should_toggle_spool(self):
"""
Test do_spool(self, line)
@@ -137,30 +121,33 @@ class TestSpiderFootCli(unittest.TestCase):
sfcli.ownopts['cli.spool_file'] = '/dev/null'
- sfcli.do_spool(0)
+ sfcli.do_spool(None)
initial_spool_state = sfcli.ownopts['cli.spool']
- sfcli.do_spool(1)
+ sfcli.do_spool(None)
new_spool_state = sfcli.ownopts['cli.spool']
self.assertNotEqual(initial_spool_state, new_spool_state)
- @unittest.skip("todo")
- def test_do_history(self):
+ def test_do_history_should_toggle_history_option(self):
"""
Test do_history(self, line)
"""
sfcli = SpiderFootCli(self.cli_default_options)
- sfcli.do_history(None)
+ sfcli.do_history("0")
+ initial_history_state = sfcli.ownopts['cli.history']
+ sfcli.do_history("1")
+ new_history_state = sfcli.ownopts['cli.history']
- self.assertEqual('TBD', 'TBD')
+ self.assertNotEqual(initial_history_state, new_history_state)
- @unittest.skip("todo")
- def test_precmd_should_print_line_to_history_file(self):
+ def test_precmd_should_return_line(self):
"""
Test precmd(self, line)
"""
sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.history'] = False
+ sfcli.ownopts['cli.spool'] = False
line = "example line"
@@ -168,14 +155,14 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual(line, precmd)
- self.assertEqual('TBD', 'TBD')
-
@unittest.skip("todo")
- def test_precmd_should_print_line_to_spool_file(self):
+ def test_precmd_should_print_line_to_history_file(self):
"""
Test precmd(self, line)
"""
sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.history'] = True
+ sfcli.ownopts['cli.spool'] = False
line = "example line"
@@ -186,11 +173,14 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
@unittest.skip("todo")
- def test_precmd_should_return_line(self):
+ def test_precmd_should_print_line_to_spool_file(self):
"""
Test precmd(self, line)
"""
sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.history'] = False
+ sfcli.ownopts['cli.spool'] = True
+ sfcli.ownopts['cli.spool_file'] = '/dev/null'
line = "example line"
@@ -198,12 +188,47 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual(line, precmd)
- @unittest.skip("todo")
- def test_ddprint(self):
+ self.assertEqual('TBD', 'TBD')
+
+ def test_dprint_should_print_if_debug_option_is_set(self):
+ """
+ Test dprint(self, msg, err=False, deb=False, plain=False, color=None)
+ """
+ sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.debug'] = True
+ sfcli.ownopts['cli.spool'] = False
+
+ io_output = io.StringIO()
+ sys.stdout = io_output
+ sfcli.dprint("example output")
+ sys.stdout = sys.__stdout__
+ output = io_output.getvalue()
+
+ self.assertIn("example output", output)
+
+ def test_dprint_should_not_print_unless_debug_option_is_set(self):
+ """
+ Test dprint(self, msg, err=False, deb=False, plain=False, color=None)
+ """
+ sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.debug'] = False
+ sfcli.ownopts['cli.spool'] = False
+
+ io_output = io.StringIO()
+ sys.stdout = io_output
+ sfcli.dprint("example output")
+ sys.stdout = sys.__stdout__
+ output = io_output.getvalue()
+
+ self.assertIn("", output)
+
+ def test_ddprint_should_print_if_debug_option_is_set(self):
"""
Test ddprint(self, msg)
"""
sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.debug'] = True
+ sfcli.ownopts['cli.spool'] = False
io_output = io.StringIO()
sys.stdout = io_output
@@ -213,13 +238,29 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertIn("example debug output", output)
- self.assertEqual('TBD', 'TBD')
+ def test_ddprint_should_not_print_unless_debug_option_is_set(self):
+ """
+ Test ddprint(self, msg)
+ """
+ sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.debug'] = False
+ sfcli.ownopts['cli.spool'] = False
- def test_edprint(self):
+ io_output = io.StringIO()
+ sys.stdout = io_output
+ sfcli.ddprint("example debug output")
+ sys.stdout = sys.__stdout__
+ output = io_output.getvalue()
+
+ self.assertEqual("", output)
+
+ def test_edprint_should_print_error_regardless_of_debug_option(self):
"""
Test edprint(self, msg)
"""
sfcli = SpiderFootCli()
+ sfcli.ownopts['cli.debug'] = False
+ sfcli.ownopts['cli.spool'] = False
io_output = io.StringIO()
sys.stdout = io_output
@@ -241,16 +282,6 @@ class TestSpiderFootCli(unittest.TestCase):
pretty = sfcli.pretty(invalid_type)
self.assertEqual("", pretty)
- @unittest.skip("todo")
- def test_request(self):
- """
- Test request(self, url, post=None)
- """
- sfcli = SpiderFootCli()
- sfcli.request(None, None)
-
- self.assertEqual('TBD', 'TBD')
-
def test_request_invalid_url_should_return_none(self):
"""
Test request(self, url, post=None)
@@ -285,6 +316,13 @@ class TestSpiderFootCli(unittest.TestCase):
Test myparseline(self, cmdline, replace=True)
"""
sfcli = SpiderFootCli()
+ parsed_line = sfcli.myparseline(None)
+
+ self.assertEqual(len(parsed_line), 2)
+ self.assertIsInstance(parsed_line, list)
+ self.assertIsInstance(parsed_line[0], list)
+ self.assertIsInstance(parsed_line[1], list)
+
parsed_line = sfcli.myparseline("")
self.assertEqual(len(parsed_line), 2)
@@ -308,7 +346,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_query(self):
"""
Test do_query(self, line)
@@ -354,7 +391,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_scaninfo(self):
"""
Test do_scaninfo(self, line)
@@ -373,7 +409,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_data(self):
"""
Test do_data(self, line)
@@ -383,7 +418,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_export(self):
"""
Test do_export(self, line)
@@ -393,7 +427,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_logs(self):
"""
Test do_logs(self, line)
@@ -403,7 +436,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_start(self):
"""
Test do_start(self, line)
@@ -413,7 +445,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_stop(self):
"""
Test do_stop(self, line)
@@ -423,7 +454,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_search(self):
"""
Test do_search(self, line)
@@ -433,7 +463,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_find(self):
"""
Test do_find(self, line)
@@ -443,7 +472,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_summary(self):
"""
Test do_summary(self, line)
@@ -453,7 +481,6 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
def test_do_delete(self):
"""
Test do_delete(self, line)
@@ -480,15 +507,17 @@ class TestSpiderFootCli(unittest.TestCase):
self.assertEqual('TBD', 'TBD')
- @unittest.skip("todo")
- def test_do_set(self):
+ def test_do_set_should_set_option(self):
"""
Test do_set(self, line)
"""
sfcli = SpiderFootCli()
- sfcli.do_set(None)
+ sfcli.ownopts['cli.test_opt'] = None
- self.assertEqual('TBD', 'TBD')
+ sfcli.do_set('cli.test_opt = "test value"')
+ new_test_opt = sfcli.ownopts['cli.test_opt']
+
+ self.assertEqual(new_test_opt, 'test value')
def test_do_shell(self):
"""