Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(logger): harden queued stream logging (#2786)
Qin Haojieb87274a4937d
diff --git a/openviking_cli/utils/logger.py b/openviking_cli/utils/logger.pyindex 8d9ae3ee7..cf72811f5 100644--- a/openviking_cli/utils/logger.py+++ b/openviking_cli/utils/logger.py@@ -4,10 +4,13 @@Logging utilities for OpenViking."""+import atexitimport logging+import queueimport sys+import threadingfrom contextlib import contextmanager-from logging.handlers import TimedRotatingFileHandler+from logging.handlers import QueueHandler, QueueListener, TimedRotatingFileHandlerfrom pathlib import Pathfrom typing import Any, Optional, Tuplefrom uuid import uuid4@@ -64,6 +67,12 @@ _MANAGED_LOGGER_ROOTS = ("openviking", "openviking_cli", "uvicorn")_shared_log_handler: Optional[logging.Handler] = None_shared_log_handler_key: Optional[tuple[Any, ...]] = None+# QueueListeners for thread-safe stdout/stderr logging (avoids StreamHandler stalls+# on application threads when process managers redirect streams).+_std_stream_handlers: dict[str, Tuple[QueueListener, logging.Handler]] = {}+_std_stream_handlers_lock = threading.RLock()+_std_stream_atexit_registered = False+def _get_log_context() -> dict[str, Any]:"""@@ -622,6 +631,7 @@ def _load_log_config() -> Tuple[str, str, str, Optional[Any]]:log_dir.mkdir(parents=True, exist_ok=True)log_output = str(log_dir / "openviking.log")except Exception:+ config = Nonelog_level_str = "INFO"log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"log_output = "stdout"@@ -636,15 +646,64 @@ def _managed_root_name(name: str) -> Optional[str]:return None+def _stop_std_stream_listeners() -> None:+ """Stop queue listeners created for stdout/stderr logging."""+ with _std_stream_handlers_lock:+ listeners = list(_std_stream_handlers.values())+ _std_stream_handlers.clear()++ for listener, real_handler in listeners:+ try:+ listener.stop()+ except Exception:+ pass+ try:+ real_handler.close()+ except Exception:+ pass+++def _create_queued_stream_handler(log_output: str) -> QueueHandler:+ """Create a QueueHandler backed by a per-stream QueueListener."""+ global _std_stream_atexit_registered++ if log_output not in ("stdout", "stderr"):+ raise ValueError(f"unsupported stream log output: {log_output}")++ with _std_stream_handlers_lock:+ entry = _std_stream_handlers.get(log_output)+ if entry is None:+ stream = sys.stdout if log_output == "stdout" else sys.stderr+ real_handler = logging.StreamHandler(stream)+ log_queue: queue.Queue = queue.Queue(-1)+ listener = QueueListener(log_queue, real_handler, respect_handler_level=True)+ listener.start()+ _std_stream_handlers[log_output] = (listener, real_handler)+ entry = (listener, real_handler)+ if not _std_stream_atexit_registered:+ atexit.register(_stop_std_stream_listeners)+ _std_stream_atexit_registered = True++ listener, real_handler = entry+ handler = QueueHandler(listener.queue)+ handler._ov_real_handler = real_handler # type: ignore[attr-defined]+ return handler+++def _add_trace_id_filter(handler: logging.Handler) -> None:+ from openviking.telemetry.tracer import TraceIdLoggingFilter++ if not any(isinstance(filter_, TraceIdLoggingFilter) for filter_ in handler.filters):+ handler.addFilter(TraceIdLoggingFilter())++def _create_log_handler(log_output: str, config: Optional[Any]) -> logging.Handler:# Prevent creating a file literally named "file"if log_output == "file":log_output = "stdout"- if log_output == "stdout":- return logging.StreamHandler(sys.stdout)- elif log_output == "stderr":- return logging.StreamHandler(sys.stderr)+ if log_output in ("stdout", "stderr"):+ return _create_queued_stream_handler(log_output)else:if config is not None:try:@@ -681,12 +740,20 @@ def _build_standard_handler(format_string: str,) -> logging.Handler:handler = _create_log_handler(log_output, config)- handler.setFormatter(logging.Formatter(format_string))- # Add trace id filter- from openviking.telemetry.tracer import TraceIdLoggingFilter+ # QueueHandler must enrich and format records on the caller thread so+ # context-local trace data and per-logger format strings are captured before+ # the shared listener-side stream handler writes the prepared message.+ if isinstance(handler, QueueHandler):+ real = getattr(handler, "_ov_real_handler", None)+ if real is not None:+ real.setFormatter(logging.Formatter("%(message)s"))+ handler.setFormatter(logging.Formatter(format_string))+ _add_trace_id_filter(handler)+ return handler- handler.addFilter(TraceIdLoggingFilter())+ handler.setFormatter(logging.Formatter(format_string))+ _add_trace_id_filter(handler)return handlerdiff --git a/tests/test_config_loader.py b/tests/test_config_loader.pyindex b1d3e83f5..2ade6a182 100644--- a/tests/test_config_loader.py+++ b/tests/test_config_loader.py@@ -3,6 +3,7 @@"""Tests for config_loader utilities."""import logging+from logging.handlers import QueueHandlerimport pytest@@ -389,6 +390,7 @@ def test_early_logger_initialization_is_reconfigured_to_file_output(tmp_path, mologger.propagate = Truelogger_module._shared_log_handler = Nonelogger_module._shared_log_handler_key = None+ logger_module._stop_std_stream_listeners()OpenVikingConfigSingleton.reset_instance()monkeypatch.setenv("OPENVIKING_CONFIG_FILE", "/tmp/codex-no-config.json")@@ -396,7 +398,7 @@ def test_early_logger_initialization_is_reconfigured_to_file_output(tmp_path, moearly_logger = logger_module.get_logger(logger_name)openviking_root = logging.getLogger("openviking")assert early_logger.handlers == []- assert any(isinstance(h, logging.StreamHandler) for h in openviking_root.handlers)+ assert any(isinstance(h, QueueHandler) for h in openviking_root.handlers)config_path = tmp_path / "ov.conf"config_path.write_text(@@ -442,4 +444,5 @@ def test_early_logger_initialization_is_reconfigured_to_file_output(tmp_path, mologger.propagate = Truelogger_module._shared_log_handler = Nonelogger_module._shared_log_handler_key = None+ logger_module._stop_std_stream_listeners()OpenVikingConfigSingleton.reset_instance()