Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Modern structured logging configuration with WebSocket broadcasting."""
import sys
import asyncio
import structlog
import logging
from contextlib import asynccontextmanager, contextmanager
from datetime import datetime
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import Any, AsyncIterator, Dict, Iterator, Optional
from queue import Queue
from threading import Thread
from core.config import Settings
# ---------------------------------------------------------------------------
# Source-tag resolution for the Terminal UI panel
# ---------------------------------------------------------------------------
#
# Every log record's ``record.name`` is a dotted-module path
# (``services.workflow.executor``, ``nodes.telegram._service``,
# ``routers.websocket``). The Terminal panel renders a single
# ``source`` column, so we collapse the dotted path to a concise tag
# (≤12 chars suits the supervisor's Honcho-style alignment, see
# ``cli/colors.py``).
#
# Three resolution stages, in order:
#
# 1. **Plugin auto-rule** — ``nodes.<plugin>.<anything>`` → ``<plugin>``.
# Adding a new plugin folder gets the right tag for free; there is
# no per-plugin entry in this module.
# 2. **Router auto-rule** — ``routers.<name>.<anything>`` → ``<name>``.
# 3. **Explicit registry** — :data:`_LOG_SOURCE_TAGS` plus runtime
# additions via :func:`register_log_source_tag`. Only cross-cutting
# services / core infra need entries here — typically because the
# module name is too long (``workflow_validator`` → ``validator``)
# or because several submodules share one logical area
# (``services.user_auth`` + ``services.auth`` → ``auth``).
# 4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).
#
# No node-specific entries belong here. Plugins that genuinely need a
# different label from their folder name should call
# :func:`register_log_source_tag` from their package ``__init__.py``,
# in the same self-registration style as the five plugin registries
# (ws_handler, filter_builder, trigger_precheck, service_refresh,
# output_schema).
_LOG_SOURCE_TAGS: Dict[str, str] = {
# Cross-cutting service short tags. Longer prefixes must precede
# shorter parents — Python ≥3.7 preserves dict-insertion order.
"services.workflow_validator": "validator",
"services.workflow_import": "wf_import",
"services.parameter_resolver": "params",
"services.node_executor": "executor",
"services.status_broadcaster": "broadcaster",
"services.ws_handler_registry": "ws_registry",
"services.credential_registry": "credentials",
"services.user_auth": "auth",
"services.model_registry": "models",
"services.node_output_schemas": "schemas",
"services.markdown_formatter": "markdown",
"services.example_loader": "examples",
"services.skill_loader": "skills",
"services.event_waiter": "waiter",
# Core-infra short tags.
"core.credentials_database": "credentials",
"core.credential_backends": "credentials",
}
def register_log_source_tag(prefix: str, tag: str) -> None:
"""Register a short Terminal-UI tag for a logger-name prefix.
Canonical extension point — matches the self-registration pattern
used by the five plugin registries. Call from a plugin folder's
``__init__.py`` only when the auto-rule produces an unwanted tag
(e.g. ``nodes.long_plugin_name`` → want ``lpn``). Most plugins do
not need this.
"""
_LOG_SOURCE_TAGS[prefix] = tag
def _resolve_source_tag(name: str) -> str:
"""Map ``record.name`` to a short Terminal-UI tag.
Order:
1. **Explicit registry** — :data:`_LOG_SOURCE_TAGS`, including any
runtime additions made via :func:`register_log_source_tag`.
Wins over the auto-rules so a plugin can override its
folder-name default (the canonical opt-out path).
2. **Plugin auto-rule** — ``nodes.<plugin>.<...>`` → ``<plugin>``.
3. **Router auto-rule** — ``routers.<name>.<...>`` → ``<name>``.
4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).
"""
# 1. Explicit registry — first matching prefix wins.
for prefix, mapped in _LOG_SOURCE_TAGS.items():
if name == prefix or name.startswith(prefix + "."):
return mapped
# 2. Plugin auto-rule.
if name.startswith("nodes."):
parts = name.split(".", 2)
if len(parts) >= 2 and parts[1] and not parts[1].startswith("_"):
return parts[1]
# 3. Router auto-rule.
if name.startswith("routers."):
parts = name.split(".", 2)
if len(parts) >= 2:
return parts[1]
# 4. Fallback — second dotted segment.
parts = name.split(".", 2)
if len(parts) >= 2 and parts[1]:
return parts[1]
return name
class WebSocketLogHandler(logging.Handler):
"""Logging handler that broadcasts logs to WebSocket clients.
Uses a thread-safe queue to bridge sync logging with async WebSocket broadcasting.
A background thread processes the queue and uses asyncio to broadcast.
"""
_instance: Optional["WebSocketLogHandler"] = None
def __init__(self, level: int = logging.INFO):
super().__init__(level)
self._queue: Queue = Queue(maxsize=1000) # Bounded queue to prevent memory issues
self._running = False
self._thread: Optional[Thread] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
@classmethod
def get_instance(cls) -> Optional["WebSocketLogHandler"]:
"""Get the singleton instance."""
return cls._instance
def emit(self, record: logging.LogRecord) -> None:
"""Queue log record for async broadcasting."""
if not self._running:
return
try:
# Get the raw message without structlog formatting
message = record.getMessage()
# Map source name via the module-level resolver (auto-rule
# for nodes.<plugin> / routers.<name>, explicit registry for
# cross-cutting services, second-segment fallback).
source = _resolve_source_tag(record.name)
# Extract structured key-value pairs from structlog
details = None
if hasattr(record, "_logger") or hasattr(record, "positional_args"):
# Try to get extra kwargs from structlog
extra_keys = set(record.__dict__.keys()) - {
"name",
"msg",
"args",
"created",
"filename",
"funcName",
"levelname",
"levelno",
"lineno",
"module",
"msecs",
"pathname",
"process",
"processName",
"relativeCreated",
"stack_info",
"exc_info",
"exc_text",
"thread",
"threadName",
"message",
"asctime",
"positional_args",
"_logger",
}
if extra_keys:
details = {k: record.__dict__[k] for k in extra_keys if not k.startswith("_")}
# Create log entry
log_data = {
"timestamp": datetime.now().isoformat(),
"level": record.levelname.lower(),
"message": message,
"source": source,
}
# Add details if present
if details:
log_data["details"] = details
# Non-blocking put - drop if queue is full
try:
self._queue.put_nowait(log_data)
except Exception:
pass # Drop log if queue is full
except Exception:
pass # Never fail in log handler
def start(self, loop: asyncio.AbstractEventLoop) -> None:
"""Start the background thread for processing logs."""
if self._running:
return
self._loop = loop
self._running = True
self._thread = Thread(target=self._process_queue, daemon=True)
self._thread.start()
WebSocketLogHandler._instance = self
def stop(self) -> None:
"""Stop the background thread."""
self._running = False
WebSocketLogHandler._instance = None
if self._thread:
self._thread.join(timeout=1.0)
def _process_queue(self) -> None:
"""Background thread that processes log queue and broadcasts."""
while self._running:
try:
# Block for up to 0.1 seconds waiting for logs
try:
log_data = self._queue.get(timeout=0.1)
except Exception:
continue
# Schedule async broadcast on the event loop
if self._loop and self._running:
asyncio.run_coroutine_threadsafe(self._broadcast(log_data), self._loop)
except Exception:
pass # Never fail in background thread
async def _broadcast(self, log_data: Dict[str, Any]) -> None:
"""Broadcast log to WebSocket clients."""
try:
from services.status_broadcaster import get_status_broadcaster
broadcaster = get_status_broadcaster()
await broadcaster.broadcast_terminal_log(log_data)
except Exception:
pass # Don't fail if broadcaster not ready
class ExpectedLLMRetryFilter(logging.Filter):
"""Keep planned provider retries readable, without an exception traceback.
The LLM step raises a benign ApplicationError to schedule the next
attempt, which the SDK logs at DEBUG ("Completing activity as failed")
and counts as no failure. Where that record is shown, it reads as a
one-line retry notice; terminal rejections and unexpected runtime errors
keep their complete tracebacks at WARNING.
"""
def filter(self, record: logging.LogRecord) -> bool:
if record.name != "temporalio.activity" or not record.exc_info:
return True
from temporalio.exceptions import ApplicationError
error = record.exc_info[1]
if (
isinstance(error, ApplicationError)
and not error.non_retryable
and error.type in {"LLMError.rate_limit", "LLMError.server", "LLMError.timeout", "LLMError.connection"}
and error.details and isinstance(error.details[0], dict)
and error.details[0].get("retryable") is True
and error.next_retry_delay is not None
):
record.msg = "Language model request will retry: %s"
record.args = (error.message,)
record.exc_info = None
record.exc_text = None
record.stack_info = None
return True
class WorkflowTaskFailureFilter(logging.Filter):
"""Log a failed workflow task at ERROR.
The SDK logs it at WARNING ("Failed activation on workflow ..."), yet a
workflow task that fails fails again on every retry until the code
changes: the workflow is stuck, not slowed down.
"""
def filter(self, record: logging.LogRecord) -> bool:
if getattr(record, "__temporal_error_identifier", None) == "WorkflowTaskFailure":
record.levelno = logging.ERROR
record.levelname = logging.getLevelName(logging.ERROR)
return True
def configure_logging(settings: Settings) -> None:
"""Configure structured logging based on settings.
Console-mode output is deliberately timestamp-less — the supervisor
(``cli/colors.py``) prepends ``[HH:MM:SS.fff]`` to every line
it aggregates, so an inner ``TimeStamper`` would produce double-time
output. JSON mode keeps the ISO timestamp because machine
consumers (log shippers, query engines) parse it as a field.
When ``log_file`` is set, a :class:`RotatingFileHandler` is used so
long-running deployments don't fill the disk. Rotation thresholds
(``log_file_max_bytes`` / ``log_file_backup_count``) come from
:class:`Settings`.
"""
log_level_value = getattr(logging, settings.log_level.upper())
# Set up log file if specified
if settings.log_file:
log_path = Path(settings.log_file)
log_path.parent.mkdir(parents=True, exist_ok=True)
# Rotating file handler — settings own the rotation thresholds.
file_handler = RotatingFileHandler(
log_path,
maxBytes=settings.log_file_max_bytes,
backupCount=settings.log_file_backup_count,
encoding="utf-8",
)
file_handler.setLevel(log_level_value)
# Configure console handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(log_level_value)
# Configure root logger
logging.basicConfig(level=log_level_value, handlers=[console_handler, file_handler], format="%(message)s")
else:
# Console only
logging.basicConfig(
format="%(message)s",
stream=sys.stdout,
level=log_level_value,
)
# Silence noisy third-party loggers
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
activity_logger = logging.getLogger("temporalio.activity")
if not any(isinstance(item, ExpectedLLMRetryFilter) for item in activity_logger.filters):
activity_logger.addFilter(ExpectedLLMRetryFilter())
workflow_logger = logging.getLogger("temporalio.worker._workflow_instance")
if not any(isinstance(item, WorkflowTaskFailureFilter) for item in workflow_logger.filters):
workflow_logger.addFilter(WorkflowTaskFailureFilter())
# Configure structlog. ``merge_contextvars`` pulls fields bound via
# ``structlog.contextvars.bind_contextvars`` (or the :func:`log_context`
# helpers) into every log record automatically — survives
# ``asyncio.gather`` child tasks via stdlib ``contextvars``.
processors = [
structlog.contextvars.merge_contextvars,
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.processors.UnicodeDecoder(),
]
# Pick the renderer. Console mode is timestamp-less by design
# (see docstring).
if settings.log_format == "json":
processors.insert(0, structlog.processors.TimeStamper(fmt="iso"))
processors.insert(0, structlog.stdlib.add_logger_name)
processors.append(structlog.processors.JSONRenderer())
else:
processors.append(
structlog.dev.ConsoleRenderer(
colors=False, # No ANSI colors for cleaner output
pad_event=35,
exception_formatter=structlog.dev.plain_traceback,
)
)
structlog.configure(
processors=processors,
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
cache_logger_on_first_use=True,
)
def get_logger(name: str) -> structlog.BoundLogger:
"""Get a structured logger instance."""
return structlog.get_logger(name)
# ---------------------------------------------------------------------------
# Context-bound logging helpers
# ---------------------------------------------------------------------------
#
# Rather than threading ``workflow_id`` / ``node_id`` / ``execution_id``
# through every ``logger.info(..., workflow_id=...)`` callsite, callers
# bind once at the entry point and every log record in that async
# context automatically picks the fields up via
# ``structlog.contextvars.merge_contextvars`` (wired in
# :func:`configure_logging`). Stdlib ``contextvars`` propagates across
# ``await`` and ``asyncio.gather`` child tasks, so per-node executions
# spawned in parallel each inherit the right context.
#
# Two helpers — pick the one that fits the call site:
#
# - :func:`log_context` (async context manager) — for ``async with``
# blocks around async work (e.g. inside ``BaseNode.execute``).
# - :func:`log_context_sync` (sync context manager) — for sync blocks
# inside sync entry points; rare since most of the codebase is async.
@asynccontextmanager
async def log_context(**fields: Any) -> AsyncIterator[None]:
"""Bind ``fields`` to every log record emitted in this async context.
Usage::
async with log_context(workflow_id=wf_id, node_id=node_id):
await do_work() # all logs inside carry workflow_id + node_id
On ``__aexit__`` each field goes back to the value it had before, even if
the inner block raises: a node executed inside another node (a tool call)
must not wipe its parent's fields. The LLM gateway's attribution headers
read the same fields (services.gateway.attribution).
"""
with structlog.contextvars.bound_contextvars(**fields):
yield
@contextmanager
def log_context_sync(**fields: Any) -> Iterator[None]:
"""Synchronous variant of :func:`log_context` for sync callers."""
with structlog.contextvars.bound_contextvars(**fields):
yield
def log_execution_time(logger: structlog.BoundLogger, operation: str, start_time: float, end_time: float, **kwargs) -> None:
"""Log execution time with additional context."""
execution_time = end_time - start_time
logger.info("Operation completed", operation=operation, execution_time_seconds=round(execution_time, 4), **kwargs)
def log_api_call(logger: structlog.BoundLogger, provider: str, model: str, operation: str, success: bool, **kwargs) -> None:
"""Log API calls with standardized format."""
logger.info("API call completed", provider=provider, model=model, operation=operation, success=success, **kwargs)
def log_cache_operation(logger: structlog.BoundLogger, operation: str, key: str, hit: bool = None, **kwargs) -> None:
"""Log cache operations."""
log_data = {"operation": operation, "cache_key": key, **kwargs}
if hit is not None:
log_data["cache_hit"] = hit
logger.debug("Cache operation", **log_data)
# Global WebSocket log handler instance
_ws_log_handler: Optional[WebSocketLogHandler] = None
def setup_websocket_logging(loop: asyncio.AbstractEventLoop, level: int = logging.INFO) -> WebSocketLogHandler:
"""Setup and start the WebSocket log handler.
Should be called during application startup after the event loop is running.
Args:
loop: The asyncio event loop to use for broadcasting
level: Minimum log level to broadcast (default: INFO)
Returns:
The WebSocket log handler instance
"""
global _ws_log_handler
if _ws_log_handler is not None:
return _ws_log_handler
# Create handler
_ws_log_handler = WebSocketLogHandler(level=level)
# Add to root logger
root_logger = logging.getLogger()
root_logger.addHandler(_ws_log_handler)
# Start the background processing thread
_ws_log_handler.start(loop)
return _ws_log_handler
def shutdown_websocket_logging() -> None:
"""Shutdown the WebSocket log handler.
Should be called during application shutdown.
"""
global _ws_log_handler
if _ws_log_handler is None:
return
# Stop the handler
_ws_log_handler.stop()
# Remove from root logger
root_logger = logging.getLogger()
root_logger.removeHandler(_ws_log_handler)
_ws_log_handler = None
def get_websocket_log_handler() -> Optional[WebSocketLogHandler]:
"""Get the current WebSocket log handler instance."""
return _ws_log_handler