Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(logging): strip double timestamps, source-tag resolver, contextvars binding, per-plugin OTel span
trohitg679d77391b22
diff --git a/server/core/config.py b/server/core/config.pyindex b92f8ea3..848cf1ce 100644--- a/server/core/config.py+++ b/server/core/config.py@@ -191,6 +191,14 @@ class Settings(BaseSettings):log_level: str = Field(default="INFO", env="LOG_LEVEL")log_format: str = Field(default="json", env="LOG_FORMAT")log_file: Optional[str] = Field(default=None, env="LOG_FILE")+ # File-rotation knobs — only consulted when LOG_FILE is set.+ # Default ceiling = 10 MiB × 5 backups = ~50 MiB on disk.+ log_file_max_bytes: int = Field(+ default=10 * 1024 * 1024, env="LOG_FILE_MAX_BYTES", ge=1024,+ )+ log_file_backup_count: int = Field(+ default=5, env="LOG_FILE_BACKUP_COUNT", ge=0,+ )# Rate Limitingrate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")diff --git a/server/core/container.py b/server/core/container.pyindex 367f6b30..a2746c37 100644--- a/server/core/container.py+++ b/server/core/container.py@@ -1,13 +1,15 @@"""Dependency injection container for the application."""-import time as _time--_ct0 = _time.perf_counter()-def _clog(msg):- elapsed = (_time.perf_counter() - _ct0) * 1000- print(f" container: {msg} ({elapsed:.0f}ms)", flush=True)+ """Pre-logger import-boundary marker.++ Emits via ``print()``; the CLI wrapper prefixes the absolute+ timestamp, so this helper carries no inline elapsed-time anymore.+ Kept as a function rather than inlining ``print()`` so future+ routing changes (e.g. to a logger after init) hit one site.+ """+ print(f" container: {msg}", flush=True)from dependency_injector import containers, providersdiff --git a/server/core/logging.py b/server/core/logging.pyindex 7db60515..c910845b 100644--- a/server/core/logging.py+++ b/server/core/logging.py@@ -4,14 +4,121 @@ import sysimport asyncioimport structlogimport logging+from contextlib import asynccontextmanager, contextmanagerfrom datetime import datetime+from logging.handlers import RotatingFileHandlerfrom pathlib import Path-from typing import Any, Dict, Optional+from typing import Any, AsyncIterator, Dict, Iterator, Optionalfrom queue import Queuefrom threading import Threadfrom core.config import Settings+# ---------------------------------------------------------------------------+# Source-tag resolution for the Terminal UI panel+# ---------------------------------------------------------------------------+#+# Every log record's ``record.name`` is a dotted-module path+# (``services.workflow.executor``, ``nodes.telegram._service``,+# ``routers.websocket``). The Terminal panel renders a single+# ``source`` column, so we collapse the dotted path to a concise tag+# (≤12 chars suits the supervisor's Honcho-style alignment, see+# ``machina/colors.py``).+#+# Three resolution stages, in order:+#+# 1. **Plugin auto-rule** — ``nodes.<plugin>.<anything>`` → ``<plugin>``.+# Adding a new plugin folder gets the right tag for free; there is+# no per-plugin entry in this module.+# 2. **Router auto-rule** — ``routers.<name>.<anything>`` → ``<name>``.+# 3. **Explicit registry** — :data:`_LOG_SOURCE_TAGS` plus runtime+# additions via :func:`register_log_source_tag`. Only cross-cutting+# services / core infra need entries here — typically because the+# module name is too long (``workflow_validator`` → ``validator``)+# or because several submodules share one logical area+# (``services.user_auth`` + ``services.auth`` → ``auth``).+# 4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).+#+# No node-specific entries belong here. Plugins that genuinely need a+# different label from their folder name should call+# :func:`register_log_source_tag` from their package ``__init__.py``,+# in the same self-registration style as the five plugin registries+# (ws_handler, filter_builder, trigger_precheck, service_refresh,+# output_schema).++_LOG_SOURCE_TAGS: Dict[str, str] = {+ # Cross-cutting service short tags. Longer prefixes must precede+ # shorter parents — Python ≥3.7 preserves dict-insertion order.+ 'services.workflow_validator': 'validator',+ 'services.workflow_import': 'wf_import',+ 'services.parameter_resolver': 'params',+ 'services.node_executor': 'executor',+ 'services.status_broadcaster': 'broadcaster',+ 'services.ws_handler_registry': 'ws_registry',+ 'services.credential_registry': 'credentials',+ 'services.user_auth': 'auth',+ 'services.model_registry': 'models',+ 'services.node_output_schemas': 'schemas',+ 'services.markdown_formatter': 'markdown',+ 'services.example_loader': 'examples',+ 'services.skill_loader': 'skills',+ 'services.event_waiter': 'waiter',++ # Core-infra short tags.+ 'core.credentials_database': 'credentials',+ 'core.credential_backends': 'credentials',+}+++def register_log_source_tag(prefix: str, tag: str) -> None:+ """Register a short Terminal-UI tag for a logger-name prefix.++ Canonical extension point — matches the self-registration pattern+ used by the five plugin registries. Call from a plugin folder's+ ``__init__.py`` only when the auto-rule produces an unwanted tag+ (e.g. ``nodes.long_plugin_name`` → want ``lpn``). Most plugins do+ not need this.+ """+ _LOG_SOURCE_TAGS[prefix] = tag+++def _resolve_source_tag(name: str) -> str:+ """Map ``record.name`` to a short Terminal-UI tag.++ Order:++ 1. **Explicit registry** — :data:`_LOG_SOURCE_TAGS`, including any+ runtime additions made via :func:`register_log_source_tag`.+ Wins over the auto-rules so a plugin can override its+ folder-name default (the canonical opt-out path).+ 2. **Plugin auto-rule** — ``nodes.<plugin>.<...>`` → ``<plugin>``.+ 3. **Router auto-rule** — ``routers.<name>.<...>`` → ``<name>``.+ 4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).+ """+ # 1. Explicit registry — first matching prefix wins.+ for prefix, mapped in _LOG_SOURCE_TAGS.items():+ if name == prefix or name.startswith(prefix + "."):+ return mapped++ # 2. Plugin auto-rule.+ if name.startswith("nodes."):+ parts = name.split(".", 2)+ if len(parts) >= 2 and parts[1] and not parts[1].startswith("_"):+ return parts[1]++ # 3. Router auto-rule.+ if name.startswith("routers."):+ parts = name.split(".", 2)+ if len(parts) >= 2:+ return parts[1]++ # 4. Fallback — second dotted segment.+ parts = name.split(".", 2)+ if len(parts) >= 2 and parts[1]:+ return parts[1]+ return name++class WebSocketLogHandler(logging.Handler):"""Logging handler that broadcasts logs to WebSocket clients.@@ -28,22 +135,6 @@ class WebSocketLogHandler(logging.Handler):self._thread: Optional[Thread] = Noneself._loop: Optional[asyncio.AbstractEventLoop] = None- # Source name mapping for cleaner display- self._source_map = {- 'services.workflow': 'workflow',- 'services.ai': 'ai',- 'nodes.android._relay': 'android',- 'nodes.android._dispatcher': 'android',- 'nodes.android._router': 'android',- 'nodes.whatsapp._service': 'whatsapp',- 'routers.websocket': 'websocket',- 'routers.workflow': 'workflow',- 'services.execution': 'execution',- 'services.deployment': 'deployment',- '__main__': 'main',- 'main': 'main',- }-@classmethoddef get_instance(cls) -> Optional['WebSocketLogHandler']:"""Get the singleton instance."""@@ -58,12 +149,10 @@ class WebSocketLogHandler(logging.Handler):# Get the raw message without structlog formattingmessage = record.getMessage()- # Map source name- source = record.name- for prefix, mapped in self._source_map.items():- if source.startswith(prefix):- source = mapped- break+ # Map source name via the module-level resolver (auto-rule+ # for nodes.<plugin> / routers.<name>, explicit registry for+ # cross-cutting services, second-segment fallback).+ source = _resolve_source_tag(record.name)# Extract structured key-value pairs from structlogdetails = None@@ -149,24 +238,42 @@ class WebSocketLogHandler(logging.Handler):def configure_logging(settings: Settings) -> None:- """Configure structured logging based on settings."""+ """Configure structured logging based on settings.++ Console-mode output is deliberately timestamp-less — the supervisor+ (``machina/colors.py``) prepends ``[HH:MM:SS.fff]`` to every line+ it aggregates, so an inner ``TimeStamper`` would produce double-time+ output. JSON mode keeps the ISO timestamp because machine+ consumers (log shippers, query engines) parse it as a field.++ When ``log_file`` is set, a :class:`RotatingFileHandler` is used so+ long-running deployments don't fill the disk. Rotation thresholds+ (``log_file_max_bytes`` / ``log_file_backup_count``) come from+ :class:`Settings`.+ """+ log_level_value = getattr(logging, settings.log_level.upper())# Set up log file if specifiedif settings.log_file:log_path = Path(settings.log_file)log_path.parent.mkdir(parents=True, exist_ok=True)- # Configure file handler- file_handler = logging.FileHandler(log_path)- file_handler.setLevel(getattr(logging, settings.log_level.upper()))+ # Rotating file handler — settings own the rotation thresholds.+ file_handler = RotatingFileHandler(+ log_path,+ maxBytes=settings.log_file_max_bytes,+ backupCount=settings.log_file_backup_count,+ encoding="utf-8",+ )+ file_handler.setLevel(log_level_value)# Configure console handlerconsole_handler = logging.StreamHandler(sys.stdout)- console_handler.setLevel(getattr(logging, settings.log_level.upper()))+ console_handler.setLevel(log_level_value)# Configure root loggerlogging.basicConfig(- level=getattr(logging, settings.log_level.upper()),+ level=log_level_value,handlers=[console_handler, file_handler],format="%(message)s")@@ -175,15 +282,19 @@ def configure_logging(settings: Settings) -> None:logging.basicConfig(format="%(message)s",stream=sys.stdout,- level=getattr(logging, settings.log_level.upper())+ level=log_level_value,)# Silence noisy third-party loggerslogging.getLogger("httpx").setLevel(logging.WARNING)logging.getLogger("httpcore").setLevel(logging.WARNING)- # Configure structlog+ # Configure structlog. ``merge_contextvars`` pulls fields bound via+ # ``structlog.contextvars.bind_contextvars`` (or the :func:`log_context`+ # helpers) into every log record automatically — survives+ # ``asyncio.gather`` child tasks via stdlib ``contextvars``.processors = [+ structlog.contextvars.merge_contextvars,structlog.stdlib.filter_by_level,structlog.stdlib.add_log_level,structlog.stdlib.PositionalArgumentsFormatter(),@@ -192,14 +303,13 @@ def configure_logging(settings: Settings) -> None:structlog.processors.UnicodeDecoder(),]- # Add appropriate renderer based on format+ # Pick the renderer. Console mode is timestamp-less by design+ # (see docstring).if settings.log_format == "json":processors.insert(0, structlog.processors.TimeStamper(fmt="iso"))processors.insert(0, structlog.stdlib.add_logger_name)processors.append(structlog.processors.JSONRenderer())else:- # Console format with timestamp- processors.insert(0, structlog.processors.TimeStamper(fmt="%H:%M:%S"))processors.append(structlog.dev.ConsoleRenderer(colors=False, # No ANSI colors for cleaner outputpad_event=35,@@ -220,6 +330,55 @@ def get_logger(name: str) -> structlog.BoundLogger:return structlog.get_logger(name)+# ---------------------------------------------------------------------------+# Context-bound logging helpers+# ---------------------------------------------------------------------------+#+# Rather than threading ``workflow_id`` / ``node_id`` / ``execution_id``+# through every ``logger.info(..., workflow_id=...)`` callsite, callers+# bind once at the entry point and every log record in that async+# context automatically picks the fields up via+# ``structlog.contextvars.merge_contextvars`` (wired in+# :func:`configure_logging`). Stdlib ``contextvars`` propagates across+# ``await`` and ``asyncio.gather`` child tasks, so per-node executions+# spawned in parallel each inherit the right context.+#+# Two helpers — pick the one that fits the call site:+#+# - :func:`log_context` (async context manager) — for ``async with``+# blocks around async work (e.g. inside ``BaseNode.execute``).+# - :func:`log_context_sync` (sync context manager) — for sync blocks+# inside sync entry points; rare since most of the codebase is async.+++@asynccontextmanager+async def log_context(**fields: Any) -> AsyncIterator[None]:+ """Bind ``fields`` to every log record emitted in this async context.++ Usage::++ async with log_context(workflow_id=wf_id, node_id=node_id):+ await do_work() # all logs inside carry workflow_id + node_id++ Bindings clear on ``__aexit__`` even if the inner block raises.+ """+ structlog.contextvars.bind_contextvars(**fields)+ try:+ yield+ finally:+ structlog.contextvars.unbind_contextvars(*fields.keys())+++@contextmanager+def log_context_sync(**fields: Any) -> Iterator[None]:+ """Synchronous variant of :func:`log_context` for sync callers."""+ structlog.contextvars.bind_contextvars(**fields)+ try:+ yield+ finally:+ structlog.contextvars.unbind_contextvars(*fields.keys())++def log_execution_time(logger: structlog.BoundLogger, operation: str,start_time: float, end_time: float, **kwargs) -> None:"""Log execution time with additional context."""diff --git a/server/main.py b/server/main.pyindex eb9d34d2..b5abb031 100644--- a/server/main.py+++ b/server/main.py@@ -4,13 +4,17 @@ Modern FastAPI backend for React Flow workflow automation platform.Refactored with dependency injection, modular services, and clean architecture."""-import time as _time-_t0 = _time.perf_counter()-def _startup_log(msg):- elapsed = _time.perf_counter() - _t0- print(f" [{elapsed:6.2f}s] {msg}", flush=True)+ """Pre-logger boot-progress marker.++ Emits via ``print()`` so the global CLI wrapper's+ ``[HH:MM:SS.ffffff]`` prefix supplies the timestamp — we don't add+ a second one here. Used during the import phase (before+ ``configure_logging`` runs) and inside the lifespan startup hook+ for milestones; after that, regular ``logger.info`` is preferred.+ """+ print(f" {msg}", flush=True)# Performance: Install uvloop if available (Linux/macOS only)diff --git a/server/routers/workflow.py b/server/routers/workflow.pyindex e03a009e..ad1bd2c4 100644--- a/server/routers/workflow.py+++ b/server/routers/workflow.py@@ -189,10 +189,16 @@ async def execute_node_for_temporal(session_id=context.get("session_id", "temporal"),workflow_id=context.get("workflow_id"),)- print(f"[Temporal Endpoint] Node {request.node_id} result: success={result.get('success')}, error={result.get('error')}")+ logger.info(+ "Temporal endpoint node result",+ node_id=request.node_id,+ success=result.get("success"),+ error=result.get("error"),+ )return result- except Exception as e:- print(f"[Temporal Endpoint] Node {request.node_id} EXCEPTION: {str(e)}")- import traceback- traceback.print_exc()+ except Exception:+ logger.exception(+ "Temporal endpoint node execution failed",+ node_id=request.node_id,+ )raise\ No newline at end of filediff --git a/server/services/plugin/base.py b/server/services/plugin/base.pyindex 656faf19..dd488e72 100644--- a/server/services/plugin/base.py+++ b/server/services/plugin/base.py@@ -22,9 +22,10 @@ import timefrom datetime import datetime, timezonefrom typing import Any, Awaitable, Callable, ClassVar, Dict, List, Optional, Sequence, Type+from opentelemetry import tracefrom pydantic import BaseModel, ValidationError-from core.logging import get_logger+from core.logging import get_logger, log_contextfrom services.plugin.connection import Connectionfrom services.plugin.context import NodeContextfrom services.plugin.credential import Credential@@ -39,6 +40,7 @@ from services.plugin.scaling import ()logger = get_logger(__name__)+tracer = trace.get_tracer(__name__)class NodeUserError(Exception):@@ -289,9 +291,58 @@ class BaseNode:"""Universal entry point. Validate params → dispatch op →wrap result. Subclasses (TriggerNode, ToolNode) override tochange the return shape or lifetime.++ The body runs under two ambient contexts:++ - :func:`core.logging.log_context` binds ``node_id`` /+ ``node_type`` / ``workflow_id`` to every log record emitted+ while the operation runs, via ``structlog.contextvars``.+ Survives ``asyncio.gather`` child tasks (stdlib+ ``contextvars`` is task-local).+ - An OpenTelemetry span named ``node.{type}.execute`` so+ per-plugin latency / failures show up in any tracer backend+ without per-plugin instrumentation. Span attributes carry+ the same identifiers as the log context."""start_time = time.time()+ workflow_id_attr: Optional[str] = None+ if isinstance(context.raw, dict):+ workflow_id_attr = context.raw.get("workflow_id")+ log_fields: Dict[str, Any] = {+ "node_id": node_id,+ "node_type": self.type,+ }+ if workflow_id_attr is not None:+ log_fields["workflow_id"] = workflow_id_attr++ async with log_context(**log_fields):+ with tracer.start_as_current_span(+ f"node.{self.type}.execute",+ attributes={+ "node.id": node_id,+ "node.type": self.type,+ **({"workflow.id": workflow_id_attr} if workflow_id_attr else {}),+ },+ ):+ return await self._execute_body(+ node_id=node_id,+ parameters=parameters,+ context=context,+ start_time=start_time,+ )++ async def _execute_body(+ self,+ *,+ node_id: str,+ parameters: Dict[str, Any],+ context: NodeContext,+ start_time: float,+ ) -> Dict[str, Any]:+ """The actual execute pipeline — extracted so :meth:`execute`+ stays a thin shell around the ambient log-context + span. Kept+ method-private; callers should always go through :meth:`execute`."""# Stash the raw (pre-validation) parameters dict in context.raw# so plugins can recover values the Pydantic extra="ignore" policy# would drop — e.g. ``api_key`` injected by node_executor'sdiff --git a/server/services/temporal/ws_client.py b/server/services/temporal/ws_client.pyindex 35caa1ff..c05733d2 100644--- a/server/services/temporal/ws_client.py+++ b/server/services/temporal/ws_client.py@@ -26,7 +26,7 @@ logger = get_logger(__name__)_settings = Settings()WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"-print(f"[Temporal WS Client] WebSocket URL configured: {WS_URL}")+logger.info("WebSocket URL configured", url=WS_URL)class WSConnectionPool:@@ -65,7 +65,10 @@ class WSConnectionPool:connector=connector,timeout=timeout,)- print(f"[WS Pool] Created session with pool_size={self.pool_size}")+ logger.info(+ "Created shared session",+ pool_size=self.pool_size,+ )return self._session@asynccontextmanager@@ -115,7 +118,7 @@ class WSConnectionPool:try:async with self.connection() as ws:await ws.send_json(message)- print(f"[WS Pool] Sent execute_node for {node_id}")+ logger.debug("Sent execute_node", node_id=node_id)# Wait for response with matching request_idasync with asyncio.timeout(timeout):@@ -123,7 +126,11 @@ class WSConnectionPool:if msg.type == aiohttp.WSMsgType.TEXT:response = json.loads(msg.data)if response.get("request_id") == request_id:- print(f"[WS Pool] Received response for {node_id}: success={response.get('success')}")+ logger.debug(+ "Received response",+ node_id=node_id,+ success=response.get("success"),+ )return responseelif msg.type == aiohttp.WSMsgType.ERROR:raise Exception(f"WebSocket error: {ws.exception()}")@@ -141,7 +148,7 @@ class WSConnectionPool:"""Close the connection pool."""if self._session and not self._session.closed:await self._session.close()- print("[WS Pool] Session closed")+ logger.info("Session closed")# Global connection pool instance