Skip to content

Self-improving AI Employees for everyone

refactor(logging): strip double timestamps, source-tag resolver, contextvars binding, per-plugin OTel span

trohitg679d77391b22
diff --git a/server/core/config.py b/server/core/config.py
index b92f8ea3..848cf1ce 100644
--- a/server/core/config.py
+++ b/server/core/config.py
@@ -191,6 +191,14 @@ class Settings(BaseSettings):
log_level: str = Field(default="INFO", env="LOG_LEVEL")
log_format: str = Field(default="json", env="LOG_FORMAT")
log_file: Optional[str] = Field(default=None, env="LOG_FILE")
+ # File-rotation knobs — only consulted when LOG_FILE is set.
+ # Default ceiling = 10 MiB × 5 backups = ~50 MiB on disk.
+ log_file_max_bytes: int = Field(
+ default=10 * 1024 * 1024, env="LOG_FILE_MAX_BYTES", ge=1024,
+ )
+ log_file_backup_count: int = Field(
+ default=5, env="LOG_FILE_BACKUP_COUNT", ge=0,
+ )
# Rate Limiting
rate_limit_enabled: bool = Field(default=True, env="RATE_LIMIT_ENABLED")
diff --git a/server/core/container.py b/server/core/container.py
index 367f6b30..a2746c37 100644
--- a/server/core/container.py
+++ b/server/core/container.py
@@ -1,13 +1,15 @@
"""Dependency injection container for the application."""
-import time as _time
-
-_ct0 = _time.perf_counter()
-
def _clog(msg):
- elapsed = (_time.perf_counter() - _ct0) * 1000
- print(f" container: {msg} ({elapsed:.0f}ms)", flush=True)
+ """Pre-logger import-boundary marker.
+
+ Emits via ``print()``; the CLI wrapper prefixes the absolute
+ timestamp, so this helper carries no inline elapsed-time anymore.
+ Kept as a function rather than inlining ``print()`` so future
+ routing changes (e.g. to a logger after init) hit one site.
+ """
+ print(f" container: {msg}", flush=True)
from dependency_injector import containers, providers
diff --git a/server/core/logging.py b/server/core/logging.py
index 7db60515..c910845b 100644
--- a/server/core/logging.py
+++ b/server/core/logging.py
@@ -4,14 +4,121 @@ import sys
import asyncio
import structlog
import logging
+from contextlib import asynccontextmanager, contextmanager
from datetime import datetime
+from logging.handlers import RotatingFileHandler
from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, AsyncIterator, Dict, Iterator, Optional
from queue import Queue
from threading import Thread
from core.config import Settings
+# ---------------------------------------------------------------------------
+# Source-tag resolution for the Terminal UI panel
+# ---------------------------------------------------------------------------
+#
+# Every log record's ``record.name`` is a dotted-module path
+# (``services.workflow.executor``, ``nodes.telegram._service``,
+# ``routers.websocket``). The Terminal panel renders a single
+# ``source`` column, so we collapse the dotted path to a concise tag
+# (≤12 chars suits the supervisor's Honcho-style alignment, see
+# ``machina/colors.py``).
+#
+# Three resolution stages, in order:
+#
+# 1. **Plugin auto-rule** — ``nodes.<plugin>.<anything>`` → ``<plugin>``.
+# Adding a new plugin folder gets the right tag for free; there is
+# no per-plugin entry in this module.
+# 2. **Router auto-rule** — ``routers.<name>.<anything>`` → ``<name>``.
+# 3. **Explicit registry** — :data:`_LOG_SOURCE_TAGS` plus runtime
+# additions via :func:`register_log_source_tag`. Only cross-cutting
+# services / core infra need entries here — typically because the
+# module name is too long (``workflow_validator`` → ``validator``)
+# or because several submodules share one logical area
+# (``services.user_auth`` + ``services.auth`` → ``auth``).
+# 4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).
+#
+# No node-specific entries belong here. Plugins that genuinely need a
+# different label from their folder name should call
+# :func:`register_log_source_tag` from their package ``__init__.py``,
+# in the same self-registration style as the five plugin registries
+# (ws_handler, filter_builder, trigger_precheck, service_refresh,
+# output_schema).
+
+_LOG_SOURCE_TAGS: Dict[str, str] = {
+ # Cross-cutting service short tags. Longer prefixes must precede
+ # shorter parents — Python ≥3.7 preserves dict-insertion order.
+ 'services.workflow_validator': 'validator',
+ 'services.workflow_import': 'wf_import',
+ 'services.parameter_resolver': 'params',
+ 'services.node_executor': 'executor',
+ 'services.status_broadcaster': 'broadcaster',
+ 'services.ws_handler_registry': 'ws_registry',
+ 'services.credential_registry': 'credentials',
+ 'services.user_auth': 'auth',
+ 'services.model_registry': 'models',
+ 'services.node_output_schemas': 'schemas',
+ 'services.markdown_formatter': 'markdown',
+ 'services.example_loader': 'examples',
+ 'services.skill_loader': 'skills',
+ 'services.event_waiter': 'waiter',
+
+ # Core-infra short tags.
+ 'core.credentials_database': 'credentials',
+ 'core.credential_backends': 'credentials',
+}
+
+
+def register_log_source_tag(prefix: str, tag: str) -> None:
+ """Register a short Terminal-UI tag for a logger-name prefix.
+
+ Canonical extension point — matches the self-registration pattern
+ used by the five plugin registries. Call from a plugin folder's
+ ``__init__.py`` only when the auto-rule produces an unwanted tag
+ (e.g. ``nodes.long_plugin_name`` → want ``lpn``). Most plugins do
+ not need this.
+ """
+ _LOG_SOURCE_TAGS[prefix] = tag
+
+
+def _resolve_source_tag(name: str) -> str:
+ """Map ``record.name`` to a short Terminal-UI tag.
+
+ Order:
+
+ 1. **Explicit registry** — :data:`_LOG_SOURCE_TAGS`, including any
+ runtime additions made via :func:`register_log_source_tag`.
+ Wins over the auto-rules so a plugin can override its
+ folder-name default (the canonical opt-out path).
+ 2. **Plugin auto-rule** — ``nodes.<plugin>.<...>`` → ``<plugin>``.
+ 3. **Router auto-rule** — ``routers.<name>.<...>`` → ``<name>``.
+ 4. **Fallback** — second dotted segment (``services.ai`` → ``ai``).
+ """
+ # 1. Explicit registry — first matching prefix wins.
+ for prefix, mapped in _LOG_SOURCE_TAGS.items():
+ if name == prefix or name.startswith(prefix + "."):
+ return mapped
+
+ # 2. Plugin auto-rule.
+ if name.startswith("nodes."):
+ parts = name.split(".", 2)
+ if len(parts) >= 2 and parts[1] and not parts[1].startswith("_"):
+ return parts[1]
+
+ # 3. Router auto-rule.
+ if name.startswith("routers."):
+ parts = name.split(".", 2)
+ if len(parts) >= 2:
+ return parts[1]
+
+ # 4. Fallback — second dotted segment.
+ parts = name.split(".", 2)
+ if len(parts) >= 2 and parts[1]:
+ return parts[1]
+ return name
+
+
class WebSocketLogHandler(logging.Handler):
"""Logging handler that broadcasts logs to WebSocket clients.
@@ -28,22 +135,6 @@ class WebSocketLogHandler(logging.Handler):
self._thread: Optional[Thread] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
- # Source name mapping for cleaner display
- self._source_map = {
- 'services.workflow': 'workflow',
- 'services.ai': 'ai',
- 'nodes.android._relay': 'android',
- 'nodes.android._dispatcher': 'android',
- 'nodes.android._router': 'android',
- 'nodes.whatsapp._service': 'whatsapp',
- 'routers.websocket': 'websocket',
- 'routers.workflow': 'workflow',
- 'services.execution': 'execution',
- 'services.deployment': 'deployment',
- '__main__': 'main',
- 'main': 'main',
- }
-
@classmethod
def get_instance(cls) -> Optional['WebSocketLogHandler']:
"""Get the singleton instance."""
@@ -58,12 +149,10 @@ class WebSocketLogHandler(logging.Handler):
# Get the raw message without structlog formatting
message = record.getMessage()
- # Map source name
- source = record.name
- for prefix, mapped in self._source_map.items():
- if source.startswith(prefix):
- source = mapped
- break
+ # Map source name via the module-level resolver (auto-rule
+ # for nodes.<plugin> / routers.<name>, explicit registry for
+ # cross-cutting services, second-segment fallback).
+ source = _resolve_source_tag(record.name)
# Extract structured key-value pairs from structlog
details = None
@@ -149,24 +238,42 @@ class WebSocketLogHandler(logging.Handler):
def configure_logging(settings: Settings) -> None:
- """Configure structured logging based on settings."""
+ """Configure structured logging based on settings.
+
+ Console-mode output is deliberately timestamp-less — the supervisor
+ (``machina/colors.py``) prepends ``[HH:MM:SS.fff]`` to every line
+ it aggregates, so an inner ``TimeStamper`` would produce double-time
+ output. JSON mode keeps the ISO timestamp because machine
+ consumers (log shippers, query engines) parse it as a field.
+
+ When ``log_file`` is set, a :class:`RotatingFileHandler` is used so
+ long-running deployments don't fill the disk. Rotation thresholds
+ (``log_file_max_bytes`` / ``log_file_backup_count``) come from
+ :class:`Settings`.
+ """
+ log_level_value = getattr(logging, settings.log_level.upper())
# Set up log file if specified
if settings.log_file:
log_path = Path(settings.log_file)
log_path.parent.mkdir(parents=True, exist_ok=True)
- # Configure file handler
- file_handler = logging.FileHandler(log_path)
- file_handler.setLevel(getattr(logging, settings.log_level.upper()))
+ # Rotating file handler — settings own the rotation thresholds.
+ file_handler = RotatingFileHandler(
+ log_path,
+ maxBytes=settings.log_file_max_bytes,
+ backupCount=settings.log_file_backup_count,
+ encoding="utf-8",
+ )
+ file_handler.setLevel(log_level_value)
# Configure console handler
console_handler = logging.StreamHandler(sys.stdout)
- console_handler.setLevel(getattr(logging, settings.log_level.upper()))
+ console_handler.setLevel(log_level_value)
# Configure root logger
logging.basicConfig(
- level=getattr(logging, settings.log_level.upper()),
+ level=log_level_value,
handlers=[console_handler, file_handler],
format="%(message)s"
)
@@ -175,15 +282,19 @@ def configure_logging(settings: Settings) -> None:
logging.basicConfig(
format="%(message)s",
stream=sys.stdout,
- level=getattr(logging, settings.log_level.upper())
+ level=log_level_value,
)
# Silence noisy third-party loggers
logging.getLogger("httpx").setLevel(logging.WARNING)
logging.getLogger("httpcore").setLevel(logging.WARNING)
- # Configure structlog
+ # Configure structlog. ``merge_contextvars`` pulls fields bound via
+ # ``structlog.contextvars.bind_contextvars`` (or the :func:`log_context`
+ # helpers) into every log record automatically — survives
+ # ``asyncio.gather`` child tasks via stdlib ``contextvars``.
processors = [
+ structlog.contextvars.merge_contextvars,
structlog.stdlib.filter_by_level,
structlog.stdlib.add_log_level,
structlog.stdlib.PositionalArgumentsFormatter(),
@@ -192,14 +303,13 @@ def configure_logging(settings: Settings) -> None:
structlog.processors.UnicodeDecoder(),
]
- # Add appropriate renderer based on format
+ # Pick the renderer. Console mode is timestamp-less by design
+ # (see docstring).
if settings.log_format == "json":
processors.insert(0, structlog.processors.TimeStamper(fmt="iso"))
processors.insert(0, structlog.stdlib.add_logger_name)
processors.append(structlog.processors.JSONRenderer())
else:
- # Console format with timestamp
- processors.insert(0, structlog.processors.TimeStamper(fmt="%H:%M:%S"))
processors.append(structlog.dev.ConsoleRenderer(
colors=False, # No ANSI colors for cleaner output
pad_event=35,
@@ -220,6 +330,55 @@ def get_logger(name: str) -> structlog.BoundLogger:
return structlog.get_logger(name)
+# ---------------------------------------------------------------------------
+# Context-bound logging helpers
+# ---------------------------------------------------------------------------
+#
+# Rather than threading ``workflow_id`` / ``node_id`` / ``execution_id``
+# through every ``logger.info(..., workflow_id=...)`` callsite, callers
+# bind once at the entry point and every log record in that async
+# context automatically picks the fields up via
+# ``structlog.contextvars.merge_contextvars`` (wired in
+# :func:`configure_logging`). Stdlib ``contextvars`` propagates across
+# ``await`` and ``asyncio.gather`` child tasks, so per-node executions
+# spawned in parallel each inherit the right context.
+#
+# Two helpers — pick the one that fits the call site:
+#
+# - :func:`log_context` (async context manager) — for ``async with``
+# blocks around async work (e.g. inside ``BaseNode.execute``).
+# - :func:`log_context_sync` (sync context manager) — for sync blocks
+# inside sync entry points; rare since most of the codebase is async.
+
+
+@asynccontextmanager
+async def log_context(**fields: Any) -> AsyncIterator[None]:
+ """Bind ``fields`` to every log record emitted in this async context.
+
+ Usage::
+
+ async with log_context(workflow_id=wf_id, node_id=node_id):
+ await do_work() # all logs inside carry workflow_id + node_id
+
+ Bindings clear on ``__aexit__`` even if the inner block raises.
+ """
+ structlog.contextvars.bind_contextvars(**fields)
+ try:
+ yield
+ finally:
+ structlog.contextvars.unbind_contextvars(*fields.keys())
+
+
+@contextmanager
+def log_context_sync(**fields: Any) -> Iterator[None]:
+ """Synchronous variant of :func:`log_context` for sync callers."""
+ structlog.contextvars.bind_contextvars(**fields)
+ try:
+ yield
+ finally:
+ structlog.contextvars.unbind_contextvars(*fields.keys())
+
+
def log_execution_time(logger: structlog.BoundLogger, operation: str,
start_time: float, end_time: float, **kwargs) -> None:
"""Log execution time with additional context."""
diff --git a/server/main.py b/server/main.py
index eb9d34d2..b5abb031 100644
--- a/server/main.py
+++ b/server/main.py
@@ -4,13 +4,17 @@ Modern FastAPI backend for React Flow workflow automation platform.
Refactored with dependency injection, modular services, and clean architecture.
"""
-import time as _time
-_t0 = _time.perf_counter()
-
def _startup_log(msg):
- elapsed = _time.perf_counter() - _t0
- print(f" [{elapsed:6.2f}s] {msg}", flush=True)
+ """Pre-logger boot-progress marker.
+
+ Emits via ``print()`` so the global CLI wrapper's
+ ``[HH:MM:SS.ffffff]`` prefix supplies the timestamp — we don't add
+ a second one here. Used during the import phase (before
+ ``configure_logging`` runs) and inside the lifespan startup hook
+ for milestones; after that, regular ``logger.info`` is preferred.
+ """
+ print(f" {msg}", flush=True)
# Performance: Install uvloop if available (Linux/macOS only)
diff --git a/server/routers/workflow.py b/server/routers/workflow.py
index e03a009e..ad1bd2c4 100644
--- a/server/routers/workflow.py
+++ b/server/routers/workflow.py
@@ -189,10 +189,16 @@ async def execute_node_for_temporal(
session_id=context.get("session_id", "temporal"),
workflow_id=context.get("workflow_id"),
)
- print(f"[Temporal Endpoint] Node {request.node_id} result: success={result.get('success')}, error={result.get('error')}")
+ logger.info(
+ "Temporal endpoint node result",
+ node_id=request.node_id,
+ success=result.get("success"),
+ error=result.get("error"),
+ )
return result
- except Exception as e:
- print(f"[Temporal Endpoint] Node {request.node_id} EXCEPTION: {str(e)}")
- import traceback
- traceback.print_exc()
+ except Exception:
+ logger.exception(
+ "Temporal endpoint node execution failed",
+ node_id=request.node_id,
+ )
raise
\ No newline at end of file
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.py
index 656faf19..dd488e72 100644
--- a/server/services/plugin/base.py
+++ b/server/services/plugin/base.py
@@ -22,9 +22,10 @@ import time
from datetime import datetime, timezone
from typing import Any, Awaitable, Callable, ClassVar, Dict, List, Optional, Sequence, Type
+from opentelemetry import trace
from pydantic import BaseModel, ValidationError
-from core.logging import get_logger
+from core.logging import get_logger, log_context
from services.plugin.connection import Connection
from services.plugin.context import NodeContext
from services.plugin.credential import Credential
@@ -39,6 +40,7 @@ from services.plugin.scaling import (
)
logger = get_logger(__name__)
+tracer = trace.get_tracer(__name__)
class NodeUserError(Exception):
@@ -289,9 +291,58 @@ class BaseNode:
"""Universal entry point. Validate params → dispatch op →
wrap result. Subclasses (TriggerNode, ToolNode) override to
change the return shape or lifetime.
+
+ The body runs under two ambient contexts:
+
+ - :func:`core.logging.log_context` binds ``node_id`` /
+ ``node_type`` / ``workflow_id`` to every log record emitted
+ while the operation runs, via ``structlog.contextvars``.
+ Survives ``asyncio.gather`` child tasks (stdlib
+ ``contextvars`` is task-local).
+ - An OpenTelemetry span named ``node.{type}.execute`` so
+ per-plugin latency / failures show up in any tracer backend
+ without per-plugin instrumentation. Span attributes carry
+ the same identifiers as the log context.
"""
start_time = time.time()
+ workflow_id_attr: Optional[str] = None
+ if isinstance(context.raw, dict):
+ workflow_id_attr = context.raw.get("workflow_id")
+ log_fields: Dict[str, Any] = {
+ "node_id": node_id,
+ "node_type": self.type,
+ }
+ if workflow_id_attr is not None:
+ log_fields["workflow_id"] = workflow_id_attr
+
+ async with log_context(**log_fields):
+ with tracer.start_as_current_span(
+ f"node.{self.type}.execute",
+ attributes={
+ "node.id": node_id,
+ "node.type": self.type,
+ **({"workflow.id": workflow_id_attr} if workflow_id_attr else {}),
+ },
+ ):
+ return await self._execute_body(
+ node_id=node_id,
+ parameters=parameters,
+ context=context,
+ start_time=start_time,
+ )
+
+ async def _execute_body(
+ self,
+ *,
+ node_id: str,
+ parameters: Dict[str, Any],
+ context: NodeContext,
+ start_time: float,
+ ) -> Dict[str, Any]:
+ """The actual execute pipeline — extracted so :meth:`execute`
+ stays a thin shell around the ambient log-context + span. Kept
+ method-private; callers should always go through :meth:`execute`."""
# Stash the raw (pre-validation) parameters dict in context.raw
# so plugins can recover values the Pydantic extra="ignore" policy
# would drop — e.g. ``api_key`` injected by node_executor's
diff --git a/server/services/temporal/ws_client.py b/server/services/temporal/ws_client.py
index 35caa1ff..c05733d2 100644
--- a/server/services/temporal/ws_client.py
+++ b/server/services/temporal/ws_client.py
@@ -26,7 +26,7 @@ logger = get_logger(__name__)
_settings = Settings()
WS_URL = f"ws://{_settings.host}:{_settings.port}/ws/internal"
-print(f"[Temporal WS Client] WebSocket URL configured: {WS_URL}")
+logger.info("WebSocket URL configured", url=WS_URL)
class WSConnectionPool:
@@ -65,7 +65,10 @@ class WSConnectionPool:
connector=connector,
timeout=timeout,
)
- print(f"[WS Pool] Created session with pool_size={self.pool_size}")
+ logger.info(
+ "Created shared session",
+ pool_size=self.pool_size,
+ )
return self._session
@asynccontextmanager
@@ -115,7 +118,7 @@ class WSConnectionPool:
try:
async with self.connection() as ws:
await ws.send_json(message)
- print(f"[WS Pool] Sent execute_node for {node_id}")
+ logger.debug("Sent execute_node", node_id=node_id)
# Wait for response with matching request_id
async with asyncio.timeout(timeout):
@@ -123,7 +126,11 @@ class WSConnectionPool:
if msg.type == aiohttp.WSMsgType.TEXT:
response = json.loads(msg.data)
if response.get("request_id") == request_id:
- print(f"[WS Pool] Received response for {node_id}: success={response.get('success')}")
+ logger.debug(
+ "Received response",
+ node_id=node_id,
+ success=response.get("success"),
+ )
return response
elif msg.type == aiohttp.WSMsgType.ERROR:
raise Exception(f"WebSocket error: {ws.exception()}")
@@ -141,7 +148,7 @@ class WSConnectionPool:
"""Close the connection pool."""
if self._session and not self._session.closed:
await self._session.close()
- print("[WS Pool] Session closed")
+ logger.info("Session closed")
# Global connection pool instance