SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sf: Replace print functions with logging
Brendan Colesba0c2d18b748
diff --git a/sf.py b/sf.pyindex fbd455a3..87dabccb 100755--- a/sf.py+++ b/sf.py@@ -110,7 +110,7 @@ dbh = Nonedef handle_abort(signal, frame):"""handle interrupt and abort scan."""- print("[*] Aborting...")+ log.info("Aborting...")if scanId and dbh:dbh.scanInstanceSet(scanId, None, None, "ABORTED")sys.exit(-1)@@ -165,7 +165,7 @@ if __name__ == '__main__':sfConfig['__webaddr'] = addrsfConfig['__webport'] = int(port)except BaseException:- print("Invalid ip:port format.")+ log.critical("Invalid ip:port format.")sys.exit(-1)else:sfConfig['__logstdout'] = True@@ -177,7 +177,7 @@ if __name__ == '__main__':mod_dir = sft.myPath() + '/modules/'if not os.path.isdir(mod_dir):- print(f"Modules directory does not exist: {mod_dir}")+ log.critical(f"Modules directory does not exist: {mod_dir}")sys.exit(-1)for filename in os.listdir(mod_dir):@@ -207,11 +207,11 @@ if __name__ == '__main__':if hasattr(sfModules[modName]['object'], 'optdescs'):sfModules[modName]['optdescs'] = sfModules[modName]['object'].optdescsexcept BaseException as e:- print(f"Failed to load {modName}: {e}")+ log.critical(f"Failed to load {modName}: {e}")sys.exit(-1)if not sfModules:- print(f"No modules found in modules directory: {mod_dir}")+ log.critical(f"No modules found in modules directory: {mod_dir}")sys.exit(-1)# Add module info to sfConfig so it can be used by the UI@@ -224,7 +224,7 @@ if __name__ == '__main__':if not args.l:if args.modules:- print("Modules available:")+ log.info("Modules available:")for m in sorted(sfModules.keys()):if "__" in m:continue@@ -232,7 +232,7 @@ if __name__ == '__main__':sys.exit(0)if args.types:- print("Types available:")+ log.info("Types available:")typedata = dbh.eventTypes()types = dict()for r in typedata:@@ -243,27 +243,27 @@ if __name__ == '__main__':sys.exit(0)if not args.s:- print("You must specify a target when running in scan mode. Try --help for guidance.")+ log.error("You must specify a target when running in scan mode. Try --help for guidance.")sys.exit(-1)if args.x and not args.t:- print("-x can only be used with -t. Use --help for guidance.")+ log.error("-x can only be used with -t. Use --help for guidance.")sys.exit(-1)if args.x and args.m:- print("-x can only be used with -t and not with -m. Use --help for guidance.")+ log.error("-x can only be used with -t and not with -m. Use --help for guidance.")sys.exit(-1)if args.r and (args.o and args.o not in ["tab", "csv"]):- print("-r can only be used when your output format is tab or csv.")+ log.error("-r can only be used when your output format is tab or csv.")sys.exit(-1)if args.H and (args.o and args.o not in ["tab", "csv"]):- print("-H can only be used when your output format is tab or csv.")+ log.error("-H can only be used when your output format is tab or csv.")sys.exit(-1)if args.D and args.o != "csv":- print("-D can only be used when using the csv output format.")+ log.error("-D can only be used when using the csv output format.")sys.exit(-1)target = args.s@@ -276,14 +276,14 @@ if __name__ == '__main__':targetType = sf.targetType(target)if not targetType:- print("[-] Could not determine target type. Invalid target: %s" % target)+ log.error(f"Could not determine target type. Invalid target: {target}")sys.exit(-1)target = target.strip('"')modlist = list()if not args.t and not args.m:- print("WARNING: You didn't specify any modules or types, so all will be enabled.")+ log.warning("You didn't specify any modules or types, so all will be enabled.")for m in list(sfModules.keys()):if "__" in m:continue@@ -324,7 +324,7 @@ if __name__ == '__main__':sfp__stor_stdout_opts['_eventtypes'] = typesif args.f:if args.f and not args.t:- print("You can only use -f with -t. Use --help for guidance.")+ log.error("You can only use -f with -t. Use --help for guidance.")sys.exit(-1)sfp__stor_stdout_opts['_showonlyrequested'] = Trueif args.F:@@ -360,14 +360,14 @@ if __name__ == '__main__':modlist.append(mod)if len(modlist) == 0:- print("Based on your criteria, no modules were enabled.")+ log.error("Based on your criteria, no modules were enabled.")sys.exit(-1)modlist += ["sfp__stor_db", "sfp__stor_stdout"]# Run the scanif sfConfig['__logging']:- print(f"[*] Modules enabled ({len(modlist)}): {','.join(modlist)}")+ log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")cfg = sf.configUnserialize(dbh.configGet(), sfConfig)# Debug mode is a variable that gets stored to the DB, so re-apply it@@ -391,7 +391,7 @@ if __name__ == '__main__':p.daemon = Truep.start()except BaseException as e:- print(f"[-] Scan [{scanId}] failed: {e}")+ log.error(f"Scan [{scanId}] failed: {e}")sys.exit(-1)# If field headers weren't disabled, print them@@ -423,7 +423,7 @@ if __name__ == '__main__':continueif info[5] in ["ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"]:if sfConfig['__logging']:- print("[*] Scan completed with status " + info[5])+ log.info(f"Scan completed with status {info[5]}")if args.o == "json":print("]")sys.exit(0)@@ -431,7 +431,7 @@ if __name__ == '__main__':sys.exit(0)# Start the web server so you can start looking at results- print('Starting web server at %s:%s ...' % (sfConfig['__webaddr'], sfConfig['__webport']))+ log.info('Starting web server at %s:%s ...' % (sfConfig['__webaddr'], sfConfig['__webport']))cherrypy.config.update({'server.socket_host': sfConfig['__webaddr'],@@ -457,7 +457,7 @@ if __name__ == '__main__':passwd_file = sf.dataPath() + '/passwd'if os.path.isfile(passwd_file):if not os.access(passwd_file, os.R_OK):- print("Could not read passwd file. Permission denied.")+ log.error("Could not read passwd file. Permission denied.")sys.exit(-1)secrets = dict()@@ -466,20 +466,20 @@ if __name__ == '__main__':for line in pw.readlines():if ':' not in line:- print("Incorrect format of passwd file, must be username:password on each line.")+ log.error("Incorrect format of passwd file, must be username:password on each line.")sys.exit(-1)u = line.strip().split(":")[0]p = ':'.join(line.strip().split(":")[1:])if not u or not p:- print("Incorrect format of passwd file, must be username:password on each line.")+ log.error("Incorrect format of passwd file, must be username:password on each line.")sys.exit(-1)secrets[u] = pif secrets:- print("Enabling authentication based on supplied passwd file.")+ log.info("Enabling authentication based on supplied passwd file.")conf['/'] = {'tools.auth_digest.on': True,'tools.auth_digest.realm': sfConfig['__webaddr'],@@ -487,29 +487,29 @@ if __name__ == '__main__':'tools.auth_digest.key': random.SystemRandom().randint(0, 99999999)}else:- print("")- print("********************************************************************")- print("Warning: passwd file contains no passwords. Authentication disabled.")- print("********************************************************************")+ warn_msg = "\n********************************************************************\n"+ warn_msg += "Warning: passwd file contains no passwords. Authentication disabled.\n"+ warn_msg += "********************************************************************\n"+ log.warning(warn_msg)else:- print("")- print("********************************************************************")- print("Please consider adding authentication to protect this instance!")- print("Refer to https://www.spiderfoot.net/documentation/#security.")- print("********************************************************************")+ warn_msg = "\n********************************************************************\n"+ warn_msg += "Please consider adding authentication to protect this instance!\n"+ warn_msg += "Refer to https://www.spiderfoot.net/documentation/#security.\n"+ warn_msg += "********************************************************************\n"+ log.warning(warn_msg)key_path = sf.dataPath() + '/spiderfoot.key'crt_path = sf.dataPath() + '/spiderfoot.crt'if os.path.isfile(key_path) and os.path.isfile(crt_path):if not os.access(crt_path, os.R_OK):- print("Could not read spiderfoot.crt file. Permission denied.")+ log.critical("Could not read spiderfoot.crt file. Permission denied.")sys.exit(-1)if not os.access(key_path, os.R_OK):- print("Could not read spiderfoot.key file. Permission denied.")+ log.critical("Could not read spiderfoot.key file. Permission denied.")sys.exit(-1)- print("Enabling SSL based on supplied key and certificate file.")+ log.info("Enabling SSL based on supplied key and certificate file.")cherrypy.server.ssl_module = 'builtin'cherrypy.server.ssl_certificate = crt_pathcherrypy.server.ssl_private_key = key_pathdiff --git a/sflib.py b/sflib.pyindex df696426..3a6508bd 100644--- a/sflib.py+++ b/sflib.py@@ -265,15 +265,15 @@ class SpiderFoot:for [parent, _id] in parents[entity]:if parent in entities:if entity != parent:- # print("Adding entity parent: " + parent)+ # self.log.debug(f"Adding entity parent: {parent}")mapping.add((entity, parent))else:ppids = list()- # print("Checking " + parent + " for entityship.")+ # self.log.debug(f"Checking {parent} for entityship.")next_parents = get_next_parent_entities(parent, ppids)for next_parent in next_parents:if entity != next_parent:- # print("Adding next entity parent: " + next_parent)+ # self.log.debug("Adding next entity parent: {next_parent}")mapping.add((entity, next_parent))return mapping@@ -441,14 +441,9 @@ class SpiderFoot:if not self.opts['__logging']:return None- if self.dbh is None:- print(f"[Error] {message}")- else:+ if self.dbh:self._dblog("ERROR", message)- if self.opts.get('__logstdout'):- print(f"[Error] {message}")-if exception:self.log.exception(message)raise BaseException(f"Internal Error Encountered: {message}")@@ -465,15 +460,13 @@ class SpiderFoot:None"""- if self.dbh is None:- print('[Fatal] %s' % error)- else:+ if self.dbh:self._dblog("FATAL", error)- print(str(inspect.stack()))-self.log.critical(error)+ print(str(inspect.stack()))+sys.exit(-1)def status(self, message):@@ -489,12 +482,8 @@ class SpiderFoot:if not self.opts['__logging']:return None- if self.dbh is None:- print("[Status] %s" % message)- else:+ if self.dbh:self._dblog("STATUS", message)- if self.opts.get('__logstdout'):- print("[*] %s" % message)self.log.info(message)@@ -527,14 +516,10 @@ class SpiderFoot:else:modName = mod.__name__- if self.dbh is None:- print('[%s] %s' % (modName, message))- else:+ if self.dbh:self._dblog("INFO", message, modName)- if self.opts.get('__logstdout'):- print("[*] %s" % message)- self.log.info(message)+ self.log.info(f"{modName} : {message}")def debug(self, message):"""Log and print a debug message.@@ -567,11 +552,7 @@ class SpiderFoot:modName = mod.__name__if self.dbh is None:- print('[%s] %s' % (modName, message))- else:self._dblog("DEBUG", message, modName)- if self.opts.get('__logstdout'):- print("[d:%s] %s" % (modName, message))self.log.debug(f"{modName} : {message}")diff --git a/sfwebui.py b/sfwebui.pyindex a553f66a..f7e6149b 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -12,6 +12,7 @@import csvimport htmlimport json+import loggingimport multiprocessing as mpimport randomimport time@@ -38,6 +39,7 @@ class SpiderFootWebUi:config = dict()token = Nonedocroot = ''+ log = logging.getLogger(__name__)def __init__(self, config):"""Initialize web server@@ -459,12 +461,12 @@ class SpiderFootWebUi:p.daemon = Truep.start()except BaseException as e:- print("[-] Scan [%s] failed: %s" % (scanId, e))- return self.error("Scan [%s] failed: %s" % (scanId, e))+ self.log.error(f"[-] Scan [{scanId}] failed: {e}")+ return self.error(f"[-] Scan [{scanId}] failed: {e}")# Wait until the scan has initializedwhile dbh.scanInstanceGet(scanId) is None:- print("[info] Waiting for the scan to initialize...")+ self.log.info("Waiting for the scan to initialize...")time.sleep(1)raise cherrypy.HTTPRedirect(f"scaninfo?id={scanId}", status=302)@@ -510,12 +512,12 @@ class SpiderFootWebUi:p.daemon = Truep.start()except BaseException as e:- print("[-] Scan [%s] failed: %s" % (scanId, e))- return self.error("Scan [%s] failed: %s" % (scanId, e))+ self.log.error(f"[-] Scan [{scanId}] failed: {e}")+ return self.error(f"[-] Scan [{scanId}] failed: {e}")# Wait until the scan has initializedwhile dbh.scanInstanceGet(scanId) is None:- print("[info] Waiting for the scan to initialize...")+ self.log.info("Waiting for the scan to initialize...")time.sleep(1)templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)@@ -1084,13 +1086,13 @@ class SpiderFootWebUi:p.daemon = Truep.start()except BaseException as e:- print("[-] Scan [%s] failed: %s" % (scanId, e))- return self.error("Scan [%s] failed: %s" % (scanId, e))+ self.log.error(f"[-] Scan [{scanId}] failed: {e}")+ return self.error(f"[-] Scan [{scanId}] failed: {e}")# Wait until the scan has initialized# Check the database for the scan status resultswhile dbh.scanInstanceGet(scanId) is None:- print("[info] Waiting for the scan to initialize...")+ self.log.info("Waiting for the scan to initialize...")time.sleep(1)if cherrypy.request.headers and 'application/json' in cherrypy.request.headers.get('Accept'):diff --git a/test/unit/test_sf.py b/test/unit/test_sf.pyindex 133403ff..c1e6fb57 100644--- a/test/unit/test_sf.py+++ b/test/unit/test_sf.py@@ -48,50 +48,50 @@ class TestSf(unittest.TestCase):def test_modules_arg_should_print_modules_and_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-M"])- self.assertIn(b"Modules available:", out)- self.assertEqual(b"", err)+ self.assertIn(b"Modules available:", err)self.assertEqual(0, code)def test_types_arg_should_print_types_and_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-T"])- self.assertIn(b"Types available:", out)- self.assertEqual(b"", err)+ self.assertIn(b"Types available:", err)+ self.assertEqual(0, code)++ @unittest.skip("todo")+ def test_l_arg_should_start_web_server(self):+ listen = "127.0.0.1:5001"+ out, err, code = self.execute([sys.executable, "sf.py", "-l", listen])+ self.assertIn(bytes(f"Starting web server at {listen}", 'utf-8'), err)self.assertEqual(0, code)def test_run_scan_with_modules_no_target_should_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-m", ",".join(self.default_modules)])- self.assertIn(b"You must specify a target when running in scan mode", out)- self.assertEqual(b"", err)+ self.assertIn(b"You must specify a target when running in scan mode", err)self.assertEqual(255, code)def test_run_scan_with_types_no_target_should_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-t", ",".join(self.default_types)])- self.assertIn(b"You must specify a target when running in scan mode", out)- self.assertEqual(b"", err)+ self.assertIn(b"You must specify a target when running in scan mode", err)self.assertEqual(255, code)- @unittest.skip("todo - output parsing fails on Python3.8")def test_run_scan_with_invalid_module_should_run_scan_and_exit(self):module = "invalid module"out, err, code = self.execute([sys.executable, "sf.py", "-m", module, "-s", "spiderfoot.net"])- self.assertIn(b"Failed to load module:", out)- self.assertEqual(b"", err)+ self.assertIn(bytes(f"Failed to load module: {module}", 'utf-8'), err)self.assertEqual(0, code)def test_run_scan_with_invalid_type_should_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-t", "invalid type", "-s", "spiderfoot.net"])- self.assertIn(b"Based on your criteria, no modules were enabled", out)- self.assertEqual(b"", err)+ self.assertIn(b"Based on your criteria, no modules were enabled", err)self.assertEqual(255, code)def test_run_scan_with_modules_should_run_scan_and_exit(self):out, err, code = self.execute([sys.executable, "sf.py", "-m", ",".join(self.default_modules), "-s", "spiderfoot.net"])- self.assertIn(b"Scan completed with status FINISHED", out)+ self.assertIn(b"Scan completed with status FINISHED", err)self.assertEqual(0, code)for module in self.default_modules:with self.subTest(module=module):- self.assertIn(module.encode(), out)+ self.assertIn(module.encode(), err)if __name__ == '__main__':