SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Unify logging with QueueHandler (#1292)
TheTechromancer11fff7d1da12
diff --git a/sf.py b/sf.pyindex 503ef757..e63b9636 100755--- a/sf.py+++ b/sf.py@@ -21,47 +21,19 @@ import signalimport sysimport timefrom copy import deepcopy-from logging import handlersimport cherrypyimport cherrypy_corsfrom cherrypy.lib import auth_digestfrom sflib import SpiderFoot-from sfscan import SpiderFootScanner+from sfscan import startSpiderFootScannerfrom sfwebui import SpiderFootWebUifrom spiderfoot import SpiderFootHelpersfrom spiderfoot import SpiderFootDb+from spiderfoot.logger import logListenerSetup, logWorkerSetupfrom spiderfoot import __version__-log = logging.getLogger()-log.setLevel(logging.DEBUG)-log_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")--debug_handler = handlers.TimedRotatingFileHandler(- "log/spiderfoot.debug.log",- when="d",- interval=1,- backupCount=30-)-debug_handler.setLevel(logging.DEBUG)-debug_handler.setFormatter(log_format)-log.addHandler(debug_handler)--error_handler = handlers.TimedRotatingFileHandler(- "log/spiderfoot.error.log",- when="d",- interval=1,- backupCount=30-)-error_handler.setLevel(logging.WARN)-error_handler.setFormatter(log_format)-log.addHandler(error_handler)--console_handler = logging.StreamHandler(sys.stderr)-console_handler.setFormatter(log_format)-log.addHandler(console_handler)-scanId = Nonedbh = None@@ -142,15 +114,17 @@ def main():if args.debug:sfConfig['_debug'] = True- log.setLevel(logging.DEBUG)else:- log.setLevel(logging.INFO)sfConfig['_debug'] = Falseif args.q:- log.setLevel(logging.NOTSET)sfConfig['__logging'] = False+ loggingQueue = mp.Queue()+ logListenerSetup(loggingQueue, sfConfig)+ logWorkerSetup(loggingQueue)+ log = logging.getLogger(f"spiderfoot.{__name__}")+sfModules = dict()sft = SpiderFoot(sfConfig)@@ -221,20 +195,23 @@ def main():sfWebUiConfig['host'] = hostsfWebUiConfig['port'] = port- start_web_server(sfWebUiConfig, sfConfig)+ start_web_server(sfWebUiConfig, sfConfig, loggingQueue)exit(0)- start_scan(sfConfig, sfModules, args)+ start_scan(sfConfig, sfModules, args, loggingQueue)-def start_scan(sfConfig, sfModules, args):+def start_scan(sfConfig, sfModules, args, loggingQueue):"""Start scanArgs:sfConfig (dict): SpiderFoot config optionssfModules (dict): modulesargs (argparse.Namespace): command line args+ loggingQueue (Queue): main SpiderFoot logging queue"""+ log = logging.getLogger(f"spiderfoot.{__name__}")+global dbhglobal scanId@@ -413,7 +390,7 @@ def start_scan(sfConfig, sfModules, args):scanName = targetscanId = SpiderFootHelpers.genScanInstanceId()try:- p = mp.Process(target=SpiderFootScanner, args=(scanName, scanId, target, targetType, modlist, cfg))+ p = mp.Process(target=startSpiderFootScanner, args=(loggingQueue, scanName, scanId, target, targetType, modlist, cfg))p.daemon = Truep.start()except BaseException as e:@@ -436,13 +413,14 @@ def start_scan(sfConfig, sfModules, args):return-def start_web_server(sfWebUiConfig, sfConfig):+def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):"""Start the web server so you can start looking at resultsArgs:sfWebUiConfig (dict): web server optionssfConfig (dict): SpiderFoot config options"""+ log = logging.getLogger(f"spiderfoot.{__name__}")web_host = sfWebUiConfig.get('host', '127.0.0.1')web_port = sfWebUiConfig.get('port', 5001)@@ -560,7 +538,7 @@ def start_web_server(sfWebUiConfig, sfConfig):# Disable auto-reloading of contentcherrypy.engine.autoreload.unsubscribe()- cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig), script_name=web_root, config=conf)+ cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig, loggingQueue), script_name=web_root, config=conf)def handle_abort(signal, frame):@@ -570,6 +548,8 @@ def handle_abort(signal, frame):signal: TBDframe: TBD"""+ log = logging.getLogger(f"spiderfoot.{__name__}")+global dbhglobal scanIddiff --git a/sflib.py b/sflib.pyindex 071ae0a1..598c5726 100644--- a/sflib.py+++ b/sflib.py@@ -58,7 +58,6 @@ class SpiderFoot:_scanId = None_socksProxy = Noneopts = dict()- log = logging.getLogger(__name__)def __init__(self, options):"""Initialize SpiderFoot object.@@ -74,6 +73,8 @@ class SpiderFoot:self.opts = deepcopy(options)+ self.log = logging.getLogger(f"spiderfoot.{__name__}")+# This is ugly but we don't want any fetches to fail - we expect# to encounter unverified SSL certs!ssl._create_default_https_context = ssl._create_unverified_context # noqa: DUO122@@ -184,27 +185,6 @@ class SpiderFoot:return val- def _dblog(self, level, message, component=None):- """Log a scan event.-- Args:- level (str): log level- message (str): log message- component (str): component from which the log event originated-- Returns:- bool: scan event logged successfully-- Raises:- BaseException: internal error encountered attempting to access the database handler- """-- if not self.dbh:- self.log.exception(f"No database handle. Could not log event to database: {message}")- raise BaseException(f"Internal Error Encountered: {message}")-- return self.dbh.scanLogEvent(self.scanId, level, message, component)-def error(self, message):"""Print and log an error message@@ -215,10 +195,7 @@ class SpiderFoot:if not self.opts['__logging']:return- if self.dbh:- self._dblog("ERROR", message)-- self.log.error(message)+ self.log.error(message, extra={'scanId': self._scanId})def fatal(self, error):"""Print an error message and stacktrace then exit.@@ -227,10 +204,7 @@ class SpiderFoot:error (str): error message"""- if self.dbh:- self._dblog("FATAL", error)-- self.log.critical(error)+ self.log.critical(error, extra={'scanId': self._scanId})print(str(inspect.stack()))@@ -246,10 +220,7 @@ class SpiderFoot:if not self.opts['__logging']:return- if self.dbh:- self._dblog("STATUS", message)-- self.log.info(message)+ self.log.info(message, extra={'scanId': self._scanId})def info(self, message):"""Log and print an info message.@@ -277,10 +248,7 @@ class SpiderFoot:else:modName = mod.__name__- if self.dbh:- self._dblog("INFO", message, modName)-- self.log.info(f"{modName} : {message}")+ self.log.info(f"{modName} : {message}", extra={'scanId': self._scanId, 'component': modName})def debug(self, message):"""Log and print a debug message.@@ -309,10 +277,7 @@ class SpiderFoot:else:modName = mod.__name__- if self.dbh:- self._dblog("DEBUG", message, modName)-- self.log.debug(f"{modName} : {message}")+ self.log.debug(f"{modName} : {message}", extra={'scanId': self._scanId})@staticmethoddef myPath():diff --git a/sfscan.py b/sfscan.pyindex 4f096421..d2e047d8 100644--- a/sfscan.py+++ b/sfscan.py@@ -21,7 +21,12 @@ from collections import OrderedDictimport dns.resolverfrom sflib import SpiderFoot-from spiderfoot import SpiderFootDb, SpiderFootEvent, SpiderFootPlugin, SpiderFootTarget, SpiderFootHelpers+from spiderfoot import SpiderFootDb, SpiderFootEvent, SpiderFootPlugin, SpiderFootTarget, SpiderFootHelpers, logger+++def startSpiderFootScanner(loggingQueue, *args, **kwargs):+ logger.logWorkerSetup(loggingQueue)+ return SpiderFootScanner(*args, **kwargs)class SpiderFootScanner():diff --git a/sfwebui.py b/sfwebui.pyindex ae116a76..2b4846c8 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -30,7 +30,8 @@ from spiderfoot import SpiderFootDbfrom spiderfoot import SpiderFootHelpersfrom spiderfoot import __version__from sflib import SpiderFoot-from sfscan import SpiderFootScanner+from sfscan import startSpiderFootScanner+from spiderfoot.logger import logListenerSetup, logWorkerSetupmp.set_start_method("spawn", force=True)@@ -43,9 +44,8 @@ class SpiderFootWebUi:config = dict()token = Nonedocroot = ''- log = logging.getLogger(__name__)- def __init__(self, web_config, config):+ def __init__(self, web_config, config, loggingQueue=None):"""Initialize web serverArgs:@@ -76,6 +76,15 @@ class SpiderFootWebUi:sf = SpiderFoot(self.defaultConfig)self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)+ # Set up logging+ if loggingQueue is None:+ self.loggingQueue = mp.Queue()+ logListenerSetup(self.loggingQueue, self.config)+ else:+ self.loggingQueue = loggingQueue+ logWorkerSetup(self.loggingQueue)+ self.log = logging.getLogger(f"spiderfoot.{__name__}")+cherrypy.config.update({'error_page.401': self.error_page_401,'error_page.404': self.error_page_404,@@ -571,7 +580,7 @@ class SpiderFootWebUi:# Start running a new scanscanId = SpiderFootHelpers.genScanInstanceId()try:- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))p.daemon = Truep.start()except Exception as e:@@ -626,7 +635,7 @@ class SpiderFootWebUi:# Start running a new scanscanId = SpiderFootHelpers.genScanInstanceId()try:- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))p.daemon = Truep.start()except Exception as e:@@ -1201,7 +1210,7 @@ class SpiderFootWebUi:# Start running a new scanscanId = SpiderFootHelpers.genScanInstanceId()try:- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))p.daemon = Truep.start()except Exception as e:diff --git a/spiderfoot/logger.py b/spiderfoot/logger.pynew file mode 100644index 00000000..c7a554ae--- /dev/null+++ b/spiderfoot/logger.py@@ -0,0 +1,118 @@+import sys+import logging+from pathlib import Path+from contextlib import suppress+from logging.handlers import QueueHandler, QueueListener+from spiderfoot import SpiderFootDb++log_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")+++class SpiderFootSqliteLogHandler(logging.Handler):+ """Handler for logging to SQLite database++ This ensure all sqlite logging is done from a single+ process and a single database handle.+ """++ def __init__(self, opts):+ self.opts = opts+ self.dbh = None+ super().__init__()++ def emit(self, record):+ if self.dbh is None:+ # Create a new database handle when the first log record is received+ self.makeDbh()+ scanId = getattr(record, "scanId", None)+ component = getattr(record, "component", None)+ if scanId:+ level = ("STATUS" if record.levelname == "INFO" else record.levelname)+ logResult = self.dbh.scanLogEvent(scanId, level, record.getMessage(), component=component)+ if logResult is False:+ # Try to recreate database handle if insert failed+ self.makeDbh()+ self.dbh.scanLogEvent(scanId, level, record.getMessage(), component=component)++ def makeDbh(self):+ self.dbh = SpiderFootDb(self.opts)+++def logListenerSetup(loggingQueue, opts=None):+ """Create and start a SpiderFoot log listener in its own thread.+ This function should be called as soon as possible in the main+ process, or whichever process is attached to stdin/stdout.++ Args:+ loggingQueue (Queue): Queue (accepts both normal and multiprocessing queue types)+ Must be instantiated in the main process.+ opts (dict): SpiderFoot config++ Returns:+ spiderFootLogListener (str): Log listener+ """+ if opts is None:+ opts = dict()+ doLogging = opts.get("__logging", True)+ debug = opts.get("_debug", False)+ logLevel = (logging.DEBUG if debug else logging.INFO)++ # Log to terminal+ console_handler = logging.StreamHandler(sys.stderr)++ # Log debug messages to file+ log_dir = Path(__file__).parent.parent / "log"+ debug_handler = logging.handlers.TimedRotatingFileHandler(+ str(log_dir / "spiderfoot.debug.log"),+ when="d",+ interval=1,+ backupCount=30+ )++ # Log error messages to file+ error_handler = logging.handlers.TimedRotatingFileHandler(+ str(log_dir / "spiderfoot.error.log"),+ when="d",+ interval=1,+ backupCount=30+ )++ # Set log level+ console_handler.setLevel(logLevel)+ debug_handler.setLevel(logging.DEBUG)+ error_handler.setLevel(logging.WARN)++ # Set log format+ console_handler.setFormatter(log_format)+ debug_handler.setFormatter(log_format)+ error_handler.setFormatter(log_format)++ if doLogging:+ handlers = [console_handler, debug_handler, error_handler]+ else:+ handlers = []++ import atexit+ if doLogging and opts is not None:+ sqlite_handler = SpiderFootSqliteLogHandler(opts)+ sqlite_handler.setLevel(logLevel)+ sqlite_handler.setFormatter(log_format)+ handlers.append(sqlite_handler)+ spiderFootLogListener = QueueListener(loggingQueue, *handlers)+ spiderFootLogListener.start()+ atexit.register(stop_listener, spiderFootLogListener)+ return spiderFootLogListener+++def logWorkerSetup(loggingQueue):+ # Root SpiderFoot logger+ log = logging.getLogger("spiderfoot")+ log.setLevel(logging.DEBUG)+ queue_handler = QueueHandler(loggingQueue)+ log.addHandler(queue_handler)+ return log+++def stop_listener(listener):+ with suppress(Exception):+ listener.stop()diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.pyindex 4cfee85c..b32f4521 100644--- a/spiderfoot/plugin.py+++ b/spiderfoot/plugin.py@@ -23,8 +23,6 @@ class SpiderFootPlugin():socksProxy (str): SOCKS proxy"""- log = logging.getLogger(__name__)-# Will be set to True by the controller if the user aborts scanning_stopScanning = False# Modules that will be notified when this module produces events@@ -69,6 +67,8 @@ class SpiderFootPlugin():# Holds the thread object when module threading is enabledself.thread = None+ self.log = logging.getLogger(f"spiderfoot.{__name__}")+def _updateSocket(self, socksProxy):"""Hack to override module's use of socket, replacing it withone that uses the supplied SOCKS server.diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex a010fea4..69a016a9 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -107,15 +107,6 @@ class TestSpiderFoot(unittest.TestCase):opt_data = sf.optValueToData(invalid_type)self.assertEqual(opt_data, None)- def test_dblog_invalid_dbh_should_raise(self):- """- Test _dblog(self, level, message, component=None)- """- sf = SpiderFoot(self.default_options)-- with self.assertRaises(BaseException):- sf._dblog(None, None, None)-def test_error(self):"""Test error(self, error):