Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Unify logging with QueueHandler (#1292)

TheTechromancer11fff7d1da12
diff --git a/sf.py b/sf.py
index 503ef757..e63b9636 100755
--- a/sf.py
+++ b/sf.py
@@ -21,47 +21,19 @@ import signal
import sys
import time
from copy import deepcopy
-from logging import handlers
import cherrypy
import cherrypy_cors
from cherrypy.lib import auth_digest
from sflib import SpiderFoot
-from sfscan import SpiderFootScanner
+from sfscan import startSpiderFootScanner
from sfwebui import SpiderFootWebUi
from spiderfoot import SpiderFootHelpers
from spiderfoot import SpiderFootDb
+from spiderfoot.logger import logListenerSetup, logWorkerSetup
from spiderfoot import __version__
-log = logging.getLogger()
-log.setLevel(logging.DEBUG)
-log_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
-
-debug_handler = handlers.TimedRotatingFileHandler(
- "log/spiderfoot.debug.log",
- when="d",
- interval=1,
- backupCount=30
-)
-debug_handler.setLevel(logging.DEBUG)
-debug_handler.setFormatter(log_format)
-log.addHandler(debug_handler)
-
-error_handler = handlers.TimedRotatingFileHandler(
- "log/spiderfoot.error.log",
- when="d",
- interval=1,
- backupCount=30
-)
-error_handler.setLevel(logging.WARN)
-error_handler.setFormatter(log_format)
-log.addHandler(error_handler)
-
-console_handler = logging.StreamHandler(sys.stderr)
-console_handler.setFormatter(log_format)
-log.addHandler(console_handler)
-
scanId = None
dbh = None
@@ -142,15 +114,17 @@ def main():
if args.debug:
sfConfig['_debug'] = True
- log.setLevel(logging.DEBUG)
else:
- log.setLevel(logging.INFO)
sfConfig['_debug'] = False
if args.q:
- log.setLevel(logging.NOTSET)
sfConfig['__logging'] = False
+ loggingQueue = mp.Queue()
+ logListenerSetup(loggingQueue, sfConfig)
+ logWorkerSetup(loggingQueue)
+ log = logging.getLogger(f"spiderfoot.{__name__}")
+
sfModules = dict()
sft = SpiderFoot(sfConfig)
@@ -221,20 +195,23 @@ def main():
sfWebUiConfig['host'] = host
sfWebUiConfig['port'] = port
- start_web_server(sfWebUiConfig, sfConfig)
+ start_web_server(sfWebUiConfig, sfConfig, loggingQueue)
exit(0)
- start_scan(sfConfig, sfModules, args)
+ start_scan(sfConfig, sfModules, args, loggingQueue)
-def start_scan(sfConfig, sfModules, args):
+def start_scan(sfConfig, sfModules, args, loggingQueue):
"""Start scan
Args:
sfConfig (dict): SpiderFoot config options
sfModules (dict): modules
args (argparse.Namespace): command line args
+ loggingQueue (Queue): main SpiderFoot logging queue
"""
+ log = logging.getLogger(f"spiderfoot.{__name__}")
+
global dbh
global scanId
@@ -413,7 +390,7 @@ def start_scan(sfConfig, sfModules, args):
scanName = target
scanId = SpiderFootHelpers.genScanInstanceId()
try:
- p = mp.Process(target=SpiderFootScanner, args=(scanName, scanId, target, targetType, modlist, cfg))
+ p = mp.Process(target=startSpiderFootScanner, args=(loggingQueue, scanName, scanId, target, targetType, modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
@@ -436,13 +413,14 @@ def start_scan(sfConfig, sfModules, args):
return
-def start_web_server(sfWebUiConfig, sfConfig):
+def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):
"""Start the web server so you can start looking at results
Args:
sfWebUiConfig (dict): web server options
sfConfig (dict): SpiderFoot config options
"""
+ log = logging.getLogger(f"spiderfoot.{__name__}")
web_host = sfWebUiConfig.get('host', '127.0.0.1')
web_port = sfWebUiConfig.get('port', 5001)
@@ -560,7 +538,7 @@ def start_web_server(sfWebUiConfig, sfConfig):
# Disable auto-reloading of content
cherrypy.engine.autoreload.unsubscribe()
- cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig), script_name=web_root, config=conf)
+ cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig, loggingQueue), script_name=web_root, config=conf)
def handle_abort(signal, frame):
@@ -570,6 +548,8 @@ def handle_abort(signal, frame):
signal: TBD
frame: TBD
"""
+ log = logging.getLogger(f"spiderfoot.{__name__}")
+
global dbh
global scanId
diff --git a/sflib.py b/sflib.py
index 071ae0a1..598c5726 100644
--- a/sflib.py
+++ b/sflib.py
@@ -58,7 +58,6 @@ class SpiderFoot:
_scanId = None
_socksProxy = None
opts = dict()
- log = logging.getLogger(__name__)
def __init__(self, options):
"""Initialize SpiderFoot object.
@@ -74,6 +73,8 @@ class SpiderFoot:
self.opts = deepcopy(options)
+ self.log = logging.getLogger(f"spiderfoot.{__name__}")
+
# This is ugly but we don't want any fetches to fail - we expect
# to encounter unverified SSL certs!
ssl._create_default_https_context = ssl._create_unverified_context # noqa: DUO122
@@ -184,27 +185,6 @@ class SpiderFoot:
return val
- def _dblog(self, level, message, component=None):
- """Log a scan event.
-
- Args:
- level (str): log level
- message (str): log message
- component (str): component from which the log event originated
-
- Returns:
- bool: scan event logged successfully
-
- Raises:
- BaseException: internal error encountered attempting to access the database handler
- """
-
- if not self.dbh:
- self.log.exception(f"No database handle. Could not log event to database: {message}")
- raise BaseException(f"Internal Error Encountered: {message}")
-
- return self.dbh.scanLogEvent(self.scanId, level, message, component)
-
def error(self, message):
"""Print and log an error message
@@ -215,10 +195,7 @@ class SpiderFoot:
if not self.opts['__logging']:
return
- if self.dbh:
- self._dblog("ERROR", message)
-
- self.log.error(message)
+ self.log.error(message, extra={'scanId': self._scanId})
def fatal(self, error):
"""Print an error message and stacktrace then exit.
@@ -227,10 +204,7 @@ class SpiderFoot:
error (str): error message
"""
- if self.dbh:
- self._dblog("FATAL", error)
-
- self.log.critical(error)
+ self.log.critical(error, extra={'scanId': self._scanId})
print(str(inspect.stack()))
@@ -246,10 +220,7 @@ class SpiderFoot:
if not self.opts['__logging']:
return
- if self.dbh:
- self._dblog("STATUS", message)
-
- self.log.info(message)
+ self.log.info(message, extra={'scanId': self._scanId})
def info(self, message):
"""Log and print an info message.
@@ -277,10 +248,7 @@ class SpiderFoot:
else:
modName = mod.__name__
- if self.dbh:
- self._dblog("INFO", message, modName)
-
- self.log.info(f"{modName} : {message}")
+ self.log.info(f"{modName} : {message}", extra={'scanId': self._scanId, 'component': modName})
def debug(self, message):
"""Log and print a debug message.
@@ -309,10 +277,7 @@ class SpiderFoot:
else:
modName = mod.__name__
- if self.dbh:
- self._dblog("DEBUG", message, modName)
-
- self.log.debug(f"{modName} : {message}")
+ self.log.debug(f"{modName} : {message}", extra={'scanId': self._scanId})
@staticmethod
def myPath():
diff --git a/sfscan.py b/sfscan.py
index 4f096421..d2e047d8 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -21,7 +21,12 @@ from collections import OrderedDict
import dns.resolver
from sflib import SpiderFoot
-from spiderfoot import SpiderFootDb, SpiderFootEvent, SpiderFootPlugin, SpiderFootTarget, SpiderFootHelpers
+from spiderfoot import SpiderFootDb, SpiderFootEvent, SpiderFootPlugin, SpiderFootTarget, SpiderFootHelpers, logger
+
+
+def startSpiderFootScanner(loggingQueue, *args, **kwargs):
+ logger.logWorkerSetup(loggingQueue)
+ return SpiderFootScanner(*args, **kwargs)
class SpiderFootScanner():
diff --git a/sfwebui.py b/sfwebui.py
index ae116a76..2b4846c8 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -30,7 +30,8 @@ from spiderfoot import SpiderFootDb
from spiderfoot import SpiderFootHelpers
from spiderfoot import __version__
from sflib import SpiderFoot
-from sfscan import SpiderFootScanner
+from sfscan import startSpiderFootScanner
+from spiderfoot.logger import logListenerSetup, logWorkerSetup
mp.set_start_method("spawn", force=True)
@@ -43,9 +44,8 @@ class SpiderFootWebUi:
config = dict()
token = None
docroot = ''
- log = logging.getLogger(__name__)
- def __init__(self, web_config, config):
+ def __init__(self, web_config, config, loggingQueue=None):
"""Initialize web server
Args:
@@ -76,6 +76,15 @@ class SpiderFootWebUi:
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
+ # Set up logging
+ if loggingQueue is None:
+ self.loggingQueue = mp.Queue()
+ logListenerSetup(self.loggingQueue, self.config)
+ else:
+ self.loggingQueue = loggingQueue
+ logWorkerSetup(self.loggingQueue)
+ self.log = logging.getLogger(f"spiderfoot.{__name__}")
+
cherrypy.config.update({
'error_page.401': self.error_page_401,
'error_page.404': self.error_page_404,
@@ -571,7 +580,7 @@ class SpiderFootWebUi:
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
@@ -626,7 +635,7 @@ class SpiderFootWebUi:
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
@@ -1201,7 +1210,7 @@ class SpiderFootWebUi:
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
- p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
+ p = mp.Process(target=startSpiderFootScanner, args=(self.loggingQueue, scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
diff --git a/spiderfoot/logger.py b/spiderfoot/logger.py
new file mode 100644
index 00000000..c7a554ae
--- /dev/null
+++ b/spiderfoot/logger.py
@@ -0,0 +1,118 @@
+import sys
+import logging
+from pathlib import Path
+from contextlib import suppress
+from logging.handlers import QueueHandler, QueueListener
+from spiderfoot import SpiderFootDb
+
+log_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
+
+
+class SpiderFootSqliteLogHandler(logging.Handler):
+ """Handler for logging to SQLite database
+
+ This ensure all sqlite logging is done from a single
+ process and a single database handle.
+ """
+
+ def __init__(self, opts):
+ self.opts = opts
+ self.dbh = None
+ super().__init__()
+
+ def emit(self, record):
+ if self.dbh is None:
+ # Create a new database handle when the first log record is received
+ self.makeDbh()
+ scanId = getattr(record, "scanId", None)
+ component = getattr(record, "component", None)
+ if scanId:
+ level = ("STATUS" if record.levelname == "INFO" else record.levelname)
+ logResult = self.dbh.scanLogEvent(scanId, level, record.getMessage(), component=component)
+ if logResult is False:
+ # Try to recreate database handle if insert failed
+ self.makeDbh()
+ self.dbh.scanLogEvent(scanId, level, record.getMessage(), component=component)
+
+ def makeDbh(self):
+ self.dbh = SpiderFootDb(self.opts)
+
+
+def logListenerSetup(loggingQueue, opts=None):
+ """Create and start a SpiderFoot log listener in its own thread.
+ This function should be called as soon as possible in the main
+ process, or whichever process is attached to stdin/stdout.
+
+ Args:
+ loggingQueue (Queue): Queue (accepts both normal and multiprocessing queue types)
+ Must be instantiated in the main process.
+ opts (dict): SpiderFoot config
+
+ Returns:
+ spiderFootLogListener (str): Log listener
+ """
+ if opts is None:
+ opts = dict()
+ doLogging = opts.get("__logging", True)
+ debug = opts.get("_debug", False)
+ logLevel = (logging.DEBUG if debug else logging.INFO)
+
+ # Log to terminal
+ console_handler = logging.StreamHandler(sys.stderr)
+
+ # Log debug messages to file
+ log_dir = Path(__file__).parent.parent / "log"
+ debug_handler = logging.handlers.TimedRotatingFileHandler(
+ str(log_dir / "spiderfoot.debug.log"),
+ when="d",
+ interval=1,
+ backupCount=30
+ )
+
+ # Log error messages to file
+ error_handler = logging.handlers.TimedRotatingFileHandler(
+ str(log_dir / "spiderfoot.error.log"),
+ when="d",
+ interval=1,
+ backupCount=30
+ )
+
+ # Set log level
+ console_handler.setLevel(logLevel)
+ debug_handler.setLevel(logging.DEBUG)
+ error_handler.setLevel(logging.WARN)
+
+ # Set log format
+ console_handler.setFormatter(log_format)
+ debug_handler.setFormatter(log_format)
+ error_handler.setFormatter(log_format)
+
+ if doLogging:
+ handlers = [console_handler, debug_handler, error_handler]
+ else:
+ handlers = []
+
+ import atexit
+ if doLogging and opts is not None:
+ sqlite_handler = SpiderFootSqliteLogHandler(opts)
+ sqlite_handler.setLevel(logLevel)
+ sqlite_handler.setFormatter(log_format)
+ handlers.append(sqlite_handler)
+ spiderFootLogListener = QueueListener(loggingQueue, *handlers)
+ spiderFootLogListener.start()
+ atexit.register(stop_listener, spiderFootLogListener)
+ return spiderFootLogListener
+
+
+def logWorkerSetup(loggingQueue):
+ # Root SpiderFoot logger
+ log = logging.getLogger("spiderfoot")
+ log.setLevel(logging.DEBUG)
+ queue_handler = QueueHandler(loggingQueue)
+ log.addHandler(queue_handler)
+ return log
+
+
+def stop_listener(listener):
+ with suppress(Exception):
+ listener.stop()
diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py
index 4cfee85c..b32f4521 100644
--- a/spiderfoot/plugin.py
+++ b/spiderfoot/plugin.py
@@ -23,8 +23,6 @@ class SpiderFootPlugin():
socksProxy (str): SOCKS proxy
"""
- log = logging.getLogger(__name__)
-
# Will be set to True by the controller if the user aborts scanning
_stopScanning = False
# Modules that will be notified when this module produces events
@@ -69,6 +67,8 @@ class SpiderFootPlugin():
# Holds the thread object when module threading is enabled
self.thread = None
+ self.log = logging.getLogger(f"spiderfoot.{__name__}")
+
def _updateSocket(self, socksProxy):
"""Hack to override module's use of socket, replacing it with
one that uses the supplied SOCKS server.
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index a010fea4..69a016a9 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -107,15 +107,6 @@ class TestSpiderFoot(unittest.TestCase):
opt_data = sf.optValueToData(invalid_type)
self.assertEqual(opt_data, None)
- def test_dblog_invalid_dbh_should_raise(self):
- """
- Test _dblog(self, level, message, component=None)
- """
- sf = SpiderFoot(self.default_options)
-
- with self.assertRaises(BaseException):
- sf._dblog(None, None, None)
-
def test_error(self):
"""
Test error(self, error):