Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
feat(logging): structured TRACE events and end-to-end request correlation
Alishahryar129e771433701
diff --git a/.env.example b/.env.exampleindex 10b98f56..7f710722 100644--- a/.env.example+++ b/.env.example@@ -136,6 +136,11 @@ WEB_FETCH_ALLOWED_SCHEMES=http,httpsWEB_FETCH_ALLOW_PRIVATE_NETWORKS=false+# Structured TRACE logs: lines with `"trace": true` merge ingress/routing/cli/provider/egress+# stages. Conversation text is logged in those payloads (verbatim). Values under keys named+# like ``api_key`` / ``authorization`` are redacted. Raw transport payloads still require+# the LOG_RAW_* toggles below.+## Verbose diagnostics (avoid logging raw prompts / SSE bodies in production)DEBUG_PLATFORM_EDITS=falseDEBUG_SUBAGENT_STACK=false@@ -144,7 +149,7 @@ LOG_RAW_API_PAYLOADS=falseLOG_RAW_SSE_EVENTS=false# When true, log full exception text and tracebacks for unhandled errors (may leak request-derived data).LOG_API_ERROR_TRACEBACKS=false-# When true, log message/transcription text previews in messaging adapters (may leak user content).+# When true, log message/transcription text previews in messaging adapters only (handler ingress always TRACEs verbatim text separately).LOG_RAW_MESSAGING_CONTENT=false# When true, log full Claude CLI stderr, non-JSON stdout lines, and parser error text.LOG_RAW_CLI_DIAGNOSTICS=falsediff --git a/README.md b/README.mdindex fa540466..7586e909 100644--- a/README.md+++ b/README.md@@ -406,6 +406,8 @@ LOG_MESSAGING_ERROR_DETAILS=falseRaw logging flags can expose prompts, tool arguments, paths, and model output. Keep them off unless you are debugging locally.+Structured TRACE rows append fields such as `"trace": true`, `stage`, `event`, and `source` and include conversation context needed to follow Claude Code flows end-to-end. Dictionary keys resembling credentials (for example `api_key` / `authorization` values nested in structured payloads) are redacted; arbitrary prose you type into prompts may still appear verbatim.+### 6. Local Web Tools```dotenvdiff --git a/api/app.py b/api/app.pyindex d0aeda27..51f04069 100644--- a/api/app.py+++ b/api/app.py@@ -13,6 +13,7 @@ from starlette.types import Receive, Scope, Sendfrom config.logging_config import configure_loggingfrom config.settings import get_settings+from core.trace import extract_claude_session_id_from_headers, trace_eventfrom providers.exceptions import ProviderErrorfrom .admin_routes import router as admin_router@@ -95,6 +96,18 @@ def create_app(*, lifespan_enabled: bool = True) -> FastAPI:app_kwargs["lifespan"] = lifespanapp = FastAPI(**app_kwargs)+ @app.middleware("http")+ async def trace_http_correlation(request: Request, call_next):+ """Attach HTTP identifiers and optional Claude session id to logs."""+ claude_sid = extract_claude_session_id_from_headers(request.headers)+ with logger.contextualize(+ http_method=request.method,+ http_path=request.url.path,+ claude_session_id=claude_sid,+ ):+ response = await call_next(request)+ return response+# Register routesapp.include_router(admin_router)app.include_router(router)@@ -111,14 +124,16 @@ def create_app(*, lifespan_enabled: bool = True) -> FastAPI:message_summary, tool_names = summarize_request_validation_body(body)- logger.debug(- "Request validation failed: path={} query={} error_locs={} error_types={} message_summary={} tool_names={}",- request.url.path,- str(request.url.query),- [list(error.get("loc", ())) for error in exc.errors()],- [str(error.get("type", "")) for error in exc.errors()],- message_summary,- tool_names,+ trace_event(+ stage="ingress",+ event="server.request.validation_failed",+ source="api",+ path=request.url.path,+ query=dict(request.query_params),+ error_locs=[list(error.get("loc", ())) for error in exc.errors()],+ error_types=[str(error.get("type", "")) for error in exc.errors()],+ message_summary=message_summary,+ tool_names=tool_names,)return await request_validation_exception_handler(request, exc)diff --git a/api/routes.py b/api/routes.pyindex 63455de1..60494942 100644--- a/api/routes.py+++ b/api/routes.py@@ -5,6 +5,7 @@ from loguru import loggerfrom config.settings import Settingsfrom core.anthropic import get_token_count+from core.trace import trace_eventfrom providers.registry import ProviderRegistryfrom . import dependencies@@ -231,6 +232,7 @@ async def list_models(_auth=Depends(require_api_key),):"""List the model ids this proxy advertises to Claude-compatible clients."""+ trace_event(stage="ingress", event="api.models.list", source="api")registry = getattr(request.app.state, "provider_registry", None)provider_registry = registry if isinstance(registry, ProviderRegistry) else Nonereturn _build_models_list_response(settings, provider_registry)@@ -250,5 +252,11 @@ async def stop_cli(request: Request, _auth=Depends(require_api_key)):raise HTTPException(status_code=503, detail="Messaging system not initialized")count = await handler.stop_all_tasks()+ trace_event(+ stage="ingress",+ event="api.cli.stop_via_handler",+ source="api",+ cancelled_nodes=count,+ )logger.info("STOP_CLI: source=handler cancelled_count={}", count)return {"status": "stopped", "cancelled_count": count}diff --git a/api/services.py b/api/services.pyindex b122627e..9611e516 100644--- a/api/services.py+++ b/api/services.py@@ -14,6 +14,7 @@ from loguru import loggerfrom config.settings import Settingsfrom core.anthropic import get_token_count, get_user_facing_error_messagefrom core.anthropic.sse import ANTHROPIC_SSE_RESPONSE_HEADERS+from core.trace import api_messages_request_snapshot, trace_event, traced_async_streamfrom providers.base import BaseProviderfrom providers.exceptions import InvalidRequestError, ProviderError@@ -118,7 +119,12 @@ class ClaudeProxyService:input_tokens = self._token_counter(routed.request.messages, routed.request.system, routed.request.tools)- logger.info("Optimization: Handling Anthropic web server tool")+ trace_event(+ stage="routing",+ event="api.optimization.web_server_tool",+ source="api",+ model=routed.request.model,+ )egress = WebFetchEgressPolicy(allow_private_network_targets=self._settings.web_fetch_allow_private_networks,allowed_schemes=self._settings.web_fetch_allowed_scheme_set(),@@ -134,6 +140,12 @@ class ClaudeProxyService:optimized = try_optimizations(routed.request, self._settings)if optimized is not None:+ trace_event(+ stage="routing",+ event="api.optimization.short_circuit",+ source="api",+ model=routed.request.model,+ )return optimizedlogger.debug("No optimization matched, routing to provider")@@ -143,29 +155,57 @@ class ClaudeProxyService:thinking_enabled=routed.resolved.thinking_enabled,)- request_id = f"req_{uuid.uuid4().hex[:12]}"- logger.info(- "API_REQUEST: request_id={} model={} messages={}",- request_id,- routed.request.model,- len(routed.request.messages),+ trace_event(+ stage="routing",+ event="api.route.resolved",+ source="api",+ provider_id=routed.resolved.provider_id,+ provider_model=routed.resolved.provider_model,+ provider_model_ref=routed.resolved.provider_model_ref,+ gateway_model=routed.request.model,+ thinking_enabled=routed.resolved.thinking_enabled,)- if self._settings.log_raw_api_payloads:- logger.debug(- "FULL_PAYLOAD [{}]: {}", request_id, routed.request.model_dump()++ request_id = f"req_{uuid.uuid4().hex[:12]}"+ with logger.contextualize(request_id=request_id):+ trace_event(+ stage="ingress",+ event="api.request.received",+ source="api",+ message_count=len(routed.request.messages),+ snapshot=api_messages_request_snapshot(routed.request),)- input_tokens = self._token_counter(- routed.request.messages, routed.request.system, routed.request.tools- )- return anthropic_sse_streaming_response(- provider.stream_response(- routed.request,- input_tokens=input_tokens,- request_id=request_id,- thinking_enabled=routed.resolved.thinking_enabled,- ),- )+ if self._settings.log_raw_api_payloads:+ logger.debug(+ "FULL_PAYLOAD [{}]: {}", request_id, routed.request.model_dump()+ )++ input_tokens = self._token_counter(+ routed.request.messages,+ routed.request.system,+ routed.request.tools,+ )++ streamed = traced_async_stream(+ provider.stream_response(+ routed.request,+ input_tokens=input_tokens,+ request_id=request_id,+ thinking_enabled=routed.resolved.thinking_enabled,+ ),+ stage="egress",+ source="api",+ complete_event="api.response.stream_completed",+ interrupted_event="api.response.stream_interrupted",+ chunk_event=None,+ extra={+ "request_id": request_id,+ "provider_id": routed.resolved.provider_id,+ "gateway_model": routed.request.model,+ },+ )+ return anthropic_sse_streaming_response(streamed)except ProviderError:raise@@ -188,12 +228,23 @@ class ClaudeProxyService:tokens = self._token_counter(routed.request.messages, routed.request.system, routed.request.tools)- logger.info(- "COUNT_TOKENS: request_id={} model={} messages={} input_tokens={}",- request_id,- routed.request.model,- len(routed.request.messages),- tokens,+ trace_event(+ stage="routing",+ event="api.route.resolved",+ source="api",+ kind="count_tokens",+ provider_id=routed.resolved.provider_id,+ provider_model=routed.resolved.provider_model,+ provider_model_ref=routed.resolved.provider_model_ref,+ gateway_model=routed.request.model,+ )+ trace_event(+ stage="ingress",+ event="api.count_tokens.completed",+ source="api",+ message_count=len(routed.request.messages),+ input_tokens=tokens,+ snapshot=api_messages_request_snapshot(routed.request),)return TokenCountResponse(input_tokens=tokens)except ProviderError:diff --git a/cli/manager.py b/cli/manager.pyindex cb27419f..1c088177 100644--- a/cli/manager.py+++ b/cli/manager.py@@ -56,8 +56,6 @@ class CLISessionManager:self._real_to_temp: dict[str, str] = {}self._lock = asyncio.Lock()- logger.info("CLISessionManager initialized")-async def get_or_create_session(self, session_id: str | None = None) -> tuple[CLISession, str, bool]:@@ -87,7 +85,6 @@ class CLISessionManager:log_raw_cli_diagnostics=self._log_raw_cli_diagnostics,)self._pending_sessions[temp_id] = new_session- logger.info(f"Created new session: {temp_id}")return new_session, temp_id, Truediff --git a/cli/session.py b/cli/session.pyindex d0079756..bba070d0 100644--- a/cli/session.py+++ b/cli/session.py@@ -9,6 +9,8 @@ from typing import Anyfrom loguru import logger+from core.trace import trace_event+from .process_registry import register_pid, unregister_pid# Cap stderr capture so a runaway child cannot exhaust memory; pipe is still drained.@@ -136,7 +138,6 @@ class CLISession:"--dangerously-skip-permissions","--verbose",]- logger.info(f"Resuming Claude session {session_id}")else:cmd = [self.claude_bin,@@ -147,7 +148,6 @@ class CLISession:"--dangerously-skip-permissions","--verbose",]- logger.info("Starting new Claude session")if self.allowed_dirs:for d in self.allowed_dirs:@@ -157,6 +157,22 @@ class CLISession:settings_json = json.dumps({"plansDirectory": self.plans_directory})cmd.extend(["--settings", settings_json])+ trace_event(+ stage="claude_cli",+ event="claude_cli.process.launch",+ source="claude_cli",+ resume_session_id=(+ session_id+ if session_id and not session_id.startswith("pending_")+ else None+ ),+ fork_session=fork_session,+ prompt=prompt,+ cwd=self.workspace,+ claude_binary=self.claude_bin,+ cli_argv=cmd,+ )+try:self.process = await asyncio.create_subprocess_exec(*cmd,diff --git a/config/logging_config.py b/config/logging_config.pyindex 7c5d8fd7..560b198b 100644--- a/config/logging_config.py+++ b/config/logging_config.py@@ -9,14 +9,26 @@ included at top level for easy grep/filter.import jsonimport loggingimport re+import threadingfrom pathlib import Pathfrom loguru import logger_configured = False-# Context keys we promote to top-level JSON for traceability-_CONTEXT_KEYS = ("request_id", "node_id", "chat_id")+# Loguru ``logger.bind()`` key used by structured TRACE payloads; ``core/trace.py``+# uses the identical string constant ``TRACE_PAYLOAD_BINDING``.+_TRACE_PAYLOAD_BINDING = "trace_payload"++# Context keys we promote to top-level JSON for traceability / grep+_CONTEXT_KEYS = (+ "request_id",+ "node_id",+ "chat_id",+ "claude_session_id",+ "http_method",+ "http_path",+)_TELEGRAM_BOT_RE = re.compile(r"(https?://api\.telegram\.org/)bot([0-9]+:[A-Za-z0-9_-]+)(/?)",@@ -48,9 +60,16 @@ def _serialize_with_context(record) -> str:"function": record["function"],"line": record["line"],}+ trace_payload = extra.get(_TRACE_PAYLOAD_BINDING)for key in _CONTEXT_KEYS:if key in extra and extra[key] is not None:out[key] = extra[key]+ if isinstance(trace_payload, dict):+ for tk, tv in trace_payload.items():+ if tk in out:+ continue+ out[tk] = tv+ out["trace"] = Truerecord["_json"] = json.dumps(out, default=str)return "{_json}\n"@@ -58,20 +77,31 @@ def _serialize_with_context(record) -> str:class InterceptHandler(logging.Handler):"""Redirect stdlib logging to loguru."""+ def __init__(self) -> None:+ super().__init__()+ self._local = threading.local()+def emit(self, record: logging.LogRecord) -> None:+ if getattr(self._local, "active", False):+ # Avoid deadlock when nested stdlib records fire during a loguru emit.+ return+ self._local.active = Truetry:- level = logger.level(record.levelname).name- except ValueError:- level = record.levelno+ try:+ level = logger.level(record.levelname).name+ except ValueError:+ level = record.levelno- frame, depth = logging.currentframe(), 2- while frame is not None and frame.f_code.co_filename == logging.__file__:- frame = frame.f_back- depth += 1+ frame, depth = logging.currentframe(), 2+ while frame is not None and frame.f_code.co_filename == logging.__file__:+ frame = frame.f_back+ depth += 1- logger.opt(depth=depth, exception=record.exc_info).log(- level, record.getMessage()- )+ logger.opt(depth=depth, exception=record.exc_info).log(+ level, record.getMessage()+ )+ finally:+ self._local.active = Falsedef configure_logging(@@ -104,6 +134,7 @@ def configure_logging(encoding="utf-8",mode="a",rotation="50 MB",+ enqueue=True,)# Intercept stdlib logging: route all root logger output to logurudiff --git a/core/anthropic/sse.py b/core/anthropic/sse.pyindex 1a99f6c4..00d7bbf5 100644--- a/core/anthropic/sse.py+++ b/core/anthropic/sse.py@@ -187,12 +187,6 @@ class SSEBuilder:event_str = format_sse_event(event_type, data)if self._log_raw_events:logger.debug("SSE_EVENT: {} - {}", event_type, event_str.strip())- else:- logger.debug(- "SSE_EVENT: event_type={} serialized_bytes={}",- event_type,- len(event_str.encode("utf-8")),- )return event_strdef message_start(self) -> str:diff --git a/core/trace.py b/core/trace.pynew file mode 100644index 00000000..a077d889--- /dev/null+++ b/core/trace.py@@ -0,0 +1,214 @@+"""Structured TRACE events for end-to-end request / CLI / provider logging.++Emitted lines are merged into JSON log rows by ``config.logging_config``.+Conversation and Claude Code prompts are logged verbatim unless values live under+sanitized credential keys (e.g. ``api_key``, ``authorization``).+"""++from __future__ import annotations++import asyncio+from collections.abc import AsyncIterator, Mapping+from typing import Any++from loguru import logger++TRACE_PAYLOAD_BINDING = "trace_payload"++_SECRET_VALUE_KEYS = frozenset(+ k.lower()+ for k in (+ "authorization",+ "x-api-key",+ "anthropic-auth-token",+ "api_key",+ "password",+ "secret",+ "token",+ "bearer_token",+ "openapi_token",+ "nvidia-api-key",+ )+)+++def _sanitize_trace_value(obj: Any) -> Any:+ """Recursively copy JSON-like structures redacting credential-shaped keys."""+ if isinstance(obj, Mapping):+ out: dict[str, Any] = {}+ for k, v in obj.items():+ if str(k).lower() in _SECRET_VALUE_KEYS:+ out[str(k)] = "<redacted>"+ else:+ out[str(k)] = _sanitize_trace_value(v)+ return out+ if isinstance(obj, tuple | list):+ return [_sanitize_trace_value(x) for x in obj]+ return obj+++def trace_event(*, stage: str, event: str, source: str, **fields: Any) -> None:+ """Emit one structured TRACE row (merged into JSON by the log sink)."""+ payload = _sanitize_trace_value(+ {+ "stage": stage,+ "event": event,+ "source": source,+ **fields,+ },+ )+ logger.bind(trace_payload=payload).info("TRACE {}", event)+++def api_messages_request_snapshot(req: Any) -> dict[str, Any]:+ """Return a sanitized snapshot of an Anthropic ``MessagesRequest``-like body."""+ if hasattr(req, "model_dump"):+ data = req.model_dump(mode="python")+ elif isinstance(req, Mapping):+ data = dict(req)+ else:+ data = {}++ snapshot: dict[str, Any] = {}+ for key in (+ "model",+ "messages",+ "system",+ "tools",+ "tool_choice",+ "max_tokens",+ "thinking",+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ "metadata",+ "stream",+ "thinking_enabled",+ ):+ if key in data and data[key] is not None:+ snapshot[key] = data[key]+ return _sanitize_trace_value(snapshot)+++def extract_claude_session_id_from_headers(headers: Mapping[str, str]) -> str | None:+ """Best-effort session id forwarded by Claude Code / SDK via HTTP."""+ lowered = {str(k).lower(): v for k, v in headers.items() if isinstance(v, str)}+ for key in (+ "anthropic-session-id",+ "x-anthropic-session-id",+ "claude-session-id",+ "x-claude-session-id",+ ):+ candidate = lowered.get(key)+ if candidate:+ return candidate+ return None+++async def traced_async_stream(+ agen: AsyncIterator[str],+ *,+ stage: str,+ source: str,+ complete_event: str,+ interrupted_event: str,+ chunk_event: str | None = None,+ chunk_interval: int = 250,+ extra: Mapping[str, Any] | None = None,+) -> AsyncIterator[str]:+ """Emit TRACE rows when a text stream completes, fails, cancels, or periodically."""+ common = dict(extra or {})+ count = 0+ nbytes = 0+ interrupted = False+ try:+ async for chunk in agen:+ count += 1+ nbytes += len(chunk.encode("utf-8", errors="replace"))+ if chunk_event and chunk_interval > 0 and count % chunk_interval == 0:+ trace_event(+ stage=stage,+ event=chunk_event,+ source=source,+ stream_chunks_so_far=count,+ stream_bytes_so_far=nbytes,+ **common,+ )+ yield chunk+ except asyncio.CancelledError:+ interrupted = True+ trace_event(+ stage=stage,+ event=interrupted_event,+ source=source,+ stream_chunks=count,+ stream_bytes=nbytes,+ outcome="cancelled",+ **common,+ )+ raise+ except BaseExceptionGroup as grp:+ interrupted = True+ trace_event(+ stage=stage,+ event=interrupted_event,+ source=source,+ stream_chunks=count,+ stream_bytes=nbytes,+ outcome="exception_group",+ note=str(grp),+ **common,+ )+ raise+ except BaseException as exc:+ interrupted = True+ trace_event(+ stage=stage,+ event=interrupted_event,+ source=source,+ stream_chunks=count,+ stream_bytes=nbytes,+ outcome="error",+ exc_type=type(exc).__name__,+ **common,+ )+ raise++ if not interrupted:+ trace_event(+ stage=stage,+ event=complete_event,+ source=source,+ stream_chunks=count,+ stream_bytes=nbytes,+ outcome="ok",+ **common,+ )+++def provider_chat_body_snapshot(body: Mapping[str, Any]) -> dict[str, Any]:+ """Sanitized OpenAI-compat chat body subset for traces (conversation text verbatim)."""+ keys = ("model", "messages", "tools", "tool_choice", "temperature", "max_tokens")+ snap = {k: body[k] for k in keys if k in body and body[k] is not None}+ return _sanitize_trace_value(snap)+++def provider_native_messages_body_snapshot(body: Mapping[str, Any]) -> dict[str, Any]:+ """Sanitized Anthropic Messages API body subset for traces."""+ keys = (+ "model",+ "messages",+ "system",+ "tools",+ "tool_choice",+ "max_tokens",+ "thinking",+ "metadata",+ "temperature",+ "top_p",+ "top_k",+ "stop_sequences",+ )+ snap = {k: body[k] for k in keys if k in body and body[k] is not None}+ return _sanitize_trace_value(snap)diff --git a/messaging/handler.py b/messaging/handler.pyindex 0a5e4e01..21e8997a 100644--- a/messaging/handler.py+++ b/messaging/handler.py@@ -11,6 +11,7 @@ import asynciofrom loguru import loggerfrom core.anthropic import format_user_error_preview, get_user_facing_error_message+from core.trace import trace_eventfrom .cli_event_constants import STATUS_MESSAGE_PREFIXESfrom .command_dispatcher import (@@ -102,26 +103,17 @@ class ClaudeMessageHandler:Determines if this is a new conversation or reply,creates/extends the message tree, and queues for processing."""- raw = incoming.text or ""- if self._log_raw_messaging_content:- text_preview = raw[:80]- if len(raw) > 80:- text_preview += "..."- logger.info(- "HANDLER_ENTRY: chat_id={} message_id={} reply_to={} text_preview={!r}",- incoming.chat_id,- incoming.message_id,- incoming.reply_to_message_id,- text_preview,- )- else:- logger.info(- "HANDLER_ENTRY: chat_id={} message_id={} reply_to={} text_len={}",- incoming.chat_id,- incoming.message_id,- incoming.reply_to_message_id,- len(raw),- )+ platform_name = getattr(self.platform, "name", "messaging")+ trace_event(+ stage="ingress",+ event="turn.received",+ source=platform_name,+ chat_id=incoming.chat_id,+ platform_message_id=incoming.message_id,+ reply_to_message_id=incoming.reply_to_message_id,+ thread_id=getattr(incoming, "message_thread_id", None),+ message_text=incoming.text or "",+ )with logger.contextualize(chat_id=incoming.chat_id, node_id=incoming.message_id@@ -240,8 +232,16 @@ class ClaudeMessageHandler:)if was_queued and status_msg_id:- # Update status to show queue positionqueue_size = self.tree_queue.get_queue_size(node_id)+ trace_event(+ stage="routing",+ event="turn.queued",+ source=getattr(self.platform, "name", "messaging"),+ chat_id=incoming.chat_id,+ platform_message_id=node_id,+ status_message_id=status_msg_id,+ queue_size=queue_size,+ )await self.platform.queue_edit_message(incoming.chat_id,status_msg_id,@@ -343,10 +343,18 @@ class ClaudeMessageHandler:last_status: str | None = Noneparent_session_id = None+ platform_nm = getattr(self.platform, "name", "messaging")if tree and node.parent_id:parent_session_id = tree.get_parent_session_id(node_id)if parent_session_id:- logger.info(f"Will fork from parent session: {parent_session_id}")+ trace_event(+ stage="claude_cli",+ event="claude_cli.fork.from_parent_session",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ parent_session_id=parent_session_id,+ )editor = ThrottledTranscriptEditor(platform=self.platform,@@ -377,6 +385,33 @@ class ClaudeMessageHandler:temp_session_id = session_or_temp_idelse:captured_session_id = session_or_temp_id++ sess_evt = (+ "claude_cli.session.pending_created"+ if is_new+ else "claude_cli.session.reused"+ )+ trace_event(+ stage="claude_cli",+ event=sess_evt,+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ status_message_id=status_msg_id,+ session_handle=str(session_or_temp_id),+ parent_resume_session_id=parent_session_id,+ fork_requested=bool(parent_session_id),+ )+ trace_event(+ stage="claude_cli",+ event="claude_cli.request.sent",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ prompt=incoming.text,+ fork_session_arg=bool(parent_session_id),+ resume_session_arg=parent_session_id,+ )except RuntimeError as e:error_message = get_user_facing_error_message(e)transcript.apply({"type": "error", "message": error_message})@@ -390,10 +425,15 @@ class ClaudeMessageHandler:MessageState.ERROR,error_message=error_message,)+ trace_event(+ stage="claude_cli",+ event="claude_cli.session.limit_reached",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ )return- logger.info(f"HANDLER: Starting CLI task processing for node {node_id}")- event_count = 0async for event_data in cli_session.start_task(incoming.text,session_id=parent_session_id,@@ -404,9 +444,6 @@ class ClaudeMessageHandler:f"HANDLER: Non-dict event received: {type(event_data)}")continue- event_count += 1- if event_count % 10 == 0:- logger.debug(f"HANDLER: Processed {event_count} events so far")(captured_session_id,@@ -426,7 +463,6 @@ class ClaudeMessageHandler:parsed_list = parse_cli_event(event_data, log_raw_cli=self._log_raw_cli_diagnostics)- logger.debug(f"HANDLER: Parsed {len(parsed_list)} events from CLI")for parsed in parsed_list:(@@ -448,6 +484,13 @@ class ClaudeMessageHandler:)except asyncio.CancelledError:+ trace_event(+ stage="claude_cli",+ event="turn.processor.cancelled",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ )logger.warning(f"HANDLER: Task cancelled for node {node_id}")cancel_reason = Noneif isinstance(node.context, dict):@@ -466,6 +509,14 @@ class ClaudeMessageHandler:node_id, MessageState.ERROR, error_message="Cancelled by user")except Exception as e:+ trace_event(+ stage="claude_cli",+ event="turn.processor.exception",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ exc_type=type(e).__name__,+ )logger.error("HANDLER: Task failed with exception: {}",format_exception_for_log(@@ -480,7 +531,14 @@ class ClaudeMessageHandler:node_id, error_msg, "Parent task failed")finally:- logger.info(f"HANDLER: _process_node completed for node {node_id}")+ trace_event(+ stage="routing",+ event="turn.processor.finished",+ source=platform_nm,+ chat_id=chat_id,+ node_id=node_id,+ claude_session_id=captured_session_id or temp_session_id,+ )# Free the session-manager slot. Session IDs are persisted in the tree and# can be resumed later by ID; we don't need to keep a CLISession instance# around after this node completes.diff --git a/messaging/node_event_pipeline.py b/messaging/node_event_pipeline.pyindex 6f233e25..8481643a 100644--- a/messaging/node_event_pipeline.py+++ b/messaging/node_event_pipeline.py@@ -7,6 +7,8 @@ from typing import Anyfrom loguru import logger+from core.trace import trace_event+from .cli_event_constants import TRANSCRIPT_EVENT_TYPES, get_status_for_eventfrom .platforms.base import SessionManagerInterfacefrom .safe_diagnostics import text_len_hint@@ -34,6 +36,15 @@ async def handle_session_info_event(return captured_session_id, temp_session_idawait cli_manager.register_real_session_id(temp_session_id, real_session_id)+ trace_event(+ stage="claude_cli",+ event="claude_cli.session.registered",+ source="claude_cli",+ node_id=node_id,+ temp_session_id=temp_session_id,+ real_session_id=real_session_id,+ tree_root_id=tree.root_id if tree else None,+ )if tree and real_session_id:await tree.update_state(node_id,@@ -76,7 +87,13 @@ async def process_parsed_cli_event(elif ptype == "complete":if not had_transcript_events:transcript.apply({"type": "text_chunk", "text": "Done."})- logger.info("HANDLER: Task complete, updating UI")+ trace_event(+ stage="claude_cli",+ event="turn.completed",+ source="cli_event",+ node_id=node_id,+ claude_session_id=captured_session_id,+ )await update_ui(format_status("✅", "Complete"), force=True)if tree and captured_session_id:await tree.update_state(@@ -87,15 +104,22 @@ async def process_parsed_cli_event(session_store.save_tree(tree.root_id, tree.to_dict())elif ptype == "error":error_msg = parsed.get("message", "Unknown error")+ em = error_msg if isinstance(error_msg, str) else str(error_msg)+ trace_event(+ stage="claude_cli",+ event="turn.failed",+ source="cli_event",+ node_id=node_id,+ claude_session_id=captured_session_id,+ cli_error_message=em,+ )if log_messaging_error_details:logger.error("HANDLER: Error event received: {}", error_msg)else:- em = error_msg if isinstance(error_msg, str) else str(error_msg)logger.error("HANDLER: Error event received: message_chars={}",text_len_hint(em),)- logger.info("HANDLER: Updating UI with error status")await update_ui(format_status("❌", "Error"), force=True)if tree:await propagate_error_to_children(node_id, error_msg, "Parent task failed")diff --git a/providers/anthropic_messages.py b/providers/anthropic_messages.pyindex e63f6598..ee7ca341 100644--- a/providers/anthropic_messages.py+++ b/providers/anthropic_messages.py@@ -21,6 +21,7 @@ from core.anthropic.native_sse_block_policy import (NativeSseBlockPolicyState,transform_native_sse_block_event,)+from core.trace import provider_native_messages_body_snapshot, trace_eventfrom providers.base import BaseProvider, ProviderConfigfrom providers.error_mapping import (map_error,@@ -338,13 +339,16 @@ class AnthropicMessagesTransport(BaseProvider):body = self._build_request_body(request, thinking_enabled=thinking_enabled)thinking_enabled = self._is_thinking_enabled(request, thinking_enabled)- logger.info(- "{}_STREAM:{} natively passing Anthropic request model={} msgs={} tools={}",- tag,- req_tag,- body.get("model"),- len(body.get("messages", [])),- len(body.get("tools", [])),+ trace_event(+ stage="provider",+ event="provider.request.sent",+ source="provider",+ provider=self._provider_name,+ gateway_model=request.model,+ downstream_model=body.get("model"),+ message_count=len(body.get("messages", [])),+ tool_count=len(body.get("tools", [])),+ body=provider_native_messages_body_snapshot(body),)response: httpx.Response | None = None@@ -373,28 +377,48 @@ class AnthropicMessagesTransport(BaseProvider):_validated_stream_send)+ chunk_count = 0+ chunk_bytes = 0+async for chunk in self._iter_stream_chunks(response,state=state,thinking_enabled=thinking_enabled,):+ chunk_count += 1+ chunk_bytes += len(chunk.encode("utf-8", errors="replace"))sent_any_event = Trueemitted_tracker.feed(chunk)yield chunk+ trace_event(+ stage="provider",+ event="provider.response.completed",+ source="provider",+ provider=self._provider_name,+ gateway_model=request.model,+ sse_chunks_out=chunk_count,+ sse_bytes_out=chunk_bytes,+ )+except Exception as error:if not isinstance(error, httpx.HTTPStatusError):- self._log_stream_transport_error(tag, req_tag, error)+ self._log_stream_transport_error(+ tag, req_tag, error, request_id=request_id+ )error_message = self._get_error_message(error, request_id)if response is not None and not response.is_closed:await response.aclose()- logger.info(- "{}_STREAM: Emitting native SSE error event for {}{}",- tag,- type(error).__name__,- req_tag,+ trace_event(+ stage="provider",+ event="provider.response.error",+ source="provider",+ provider=self._provider_name,+ error_message=error_message,+ exc_type=type(error).__name__,+ mid_stream=sent_any_event,)if sent_any_event:for event in emitted_tracker.iter_close_unclosed_blocks():diff --git a/providers/base.py b/providers/base.pyindex a009ff1e..8f918ad1 100644--- a/providers/base.py+++ b/providers/base.py@@ -80,26 +80,43 @@ class BaseProvider(ABC):build(request, thinking_enabled=thinking_enabled)def _log_stream_transport_error(- self, tag: str, req_tag: str, error: Exception+ self,+ tag: str,+ req_tag: str,+ error: Exception,+ *,+ request_id: str | None = None,) -> None:"""Log streaming transport failures (metadata-only unless verbose is enabled)."""from loguru import logger+ from core.trace import trace_event++ response = getattr(error, "response", None)+ http_status = (+ getattr(response, "status_code", None) if response is not None else None+ )+ trace_event(+ stage="provider",+ event="provider.response.transport_error",+ source="provider",+ provider=tag,+ request_id=request_id,+ exc_type=type(error).__name__,+ http_status=http_status,+ )+if self._config.log_api_error_tracebacks:logger.error("{}_ERROR:{} {}: {}", tag, req_tag, type(error).__name__, error)return- response = getattr(error, "response", None)- status_code = (- getattr(response, "status_code", None) if response is not None else None- )logger.error("{}_ERROR:{} exc_type={} http_status={}",tag,req_tag,type(error).__name__,- status_code,+ http_status,)@abstractmethoddiff --git a/providers/openai_compat.py b/providers/openai_compat.pyindex 071b8276..384930e8 100644--- a/providers/openai_compat.py+++ b/providers/openai_compat.py@@ -23,6 +23,7 @@ from core.anthropic import (append_request_id,map_stop_reason,)+from core.trace import provider_chat_body_snapshot, trace_eventfrom providers.base import BaseProvider, ProviderConfigfrom providers.error_mapping import (map_error,@@ -353,13 +354,16 @@ class OpenAIChatTransport(BaseProvider):body = self._build_request_body(request, thinking_enabled=thinking_enabled)thinking_enabled = self._is_thinking_enabled(request, thinking_enabled)req_tag = f" request_id={request_id}" if request_id else ""- logger.info(- "{}_STREAM:{} model={} msgs={} tools={}",- tag,- req_tag,- body.get("model"),- len(body.get("messages", [])),- len(body.get("tools", [])),+ trace_event(+ stage="provider",+ event="provider.request.sent",+ source="provider",+ provider=self._provider_name,+ gateway_model=request.model,+ downstream_model=body.get("model"),+ message_count=len(body.get("messages", [])),+ tool_count=len(body.get("tools", [])),+ body=provider_chat_body_snapshot(body),)yield sse.message_start()@@ -455,7 +459,7 @@ class OpenAIChatTransport(BaseProvider):except asyncio.CancelledError, GeneratorExit:raiseexcept Exception as e:- self._log_stream_transport_error(tag, req_tag, e)+ self._log_stream_transport_error(tag, req_tag, e, request_id=request_id)mapped_e = map_error(e, rate_limiter=self._global_rate_limiter)base_message = user_visible_message_for_mapped_provider_error(mapped_e,@@ -463,11 +467,13 @@ class OpenAIChatTransport(BaseProvider):read_timeout_s=self._config.http_read_timeout,)error_message = append_request_id(base_message, request_id)- logger.info(- "{}_STREAM: Emitting SSE error event for {}{}",- tag,- type(e).__name__,- req_tag,+ trace_event(+ stage="provider",+ event="provider.response.error",+ source="provider",+ provider=tag,+ error_message=error_message,+ mapped_error_type=type(mapped_e).__name__,)for event in sse.close_all_blocks():yield event@@ -552,5 +558,14 @@ class OpenAIChatTransport(BaseProvider):provider_input,provider_input - input_tokens,)+ trace_event(+ stage="provider",+ event="provider.response.completed",+ source="provider",+ provider=self._provider_name,+ finish_reason=(None if finish_reason is None else str(finish_reason)),+ output_tokens=output_tokens,+ prompt_tokens_estimate=input_tokens,+ )yield sse.message_delta(map_stop_reason(finish_reason), output_tokens)yield sse.message_stop()diff --git a/tests/api/test_safe_logging.py b/tests/api/test_safe_logging.pyindex 10965168..7c3f9bc7 100644--- a/tests/api/test_safe_logging.py+++ b/tests/api/test_safe_logging.py@@ -70,11 +70,7 @@ def test_sse_builder_default_debug_has_no_serialized_json_content():sse = SSEBuilder("msg_x", "m", 1, log_raw_events=False)sse.message_start()- assert mock_debug.call_count == 1- message = str(mock_debug.call_args)- assert "serialized_bytes=" in message- assert "role" not in message- assert "assistant" not in message+ assert mock_debug.call_count == 0def test_sse_builder_raw_logging_includes_event_body_when_enabled():diff --git a/tests/core/test_trace.py b/tests/core/test_trace.pynew file mode 100644index 00000000..eb4c147d--- /dev/null+++ b/tests/core/test_trace.py@@ -0,0 +1,38 @@+"""Structured TRACE logging assertions."""++from __future__ import annotations++import json+from pathlib import Path++from loguru import logger++from config.logging_config import configure_logging+from core.trace import TRACE_PAYLOAD_BINDING, trace_event+++def test_trace_payload_merged_into_json_line(tmp_path) -> None:+ log_file = str(tmp_path / "t.log")+ configure_logging(log_file, force=True)+ trace_event(stage="s", event="e.v1", source="unit", hello="world", n=42)+ logger.complete()+ text = Path(log_file).read_text(encoding="utf-8").strip().split("\n")[-1]+ row = json.loads(text)+ assert row["trace"] is True+ assert row["stage"] == "s"+ assert row["event"] == "e.v1"+ assert row["source"] == "unit"+ assert row["hello"] == "world"+ assert row["n"] == 42+ assert TRACE_PAYLOAD_BINDING == "trace_payload"+++def test_sanitize_masks_nested_api_key_strings() -> None:+ """Credential-shaped keys redact without touching normal message text."""+ from core.trace import _sanitize_trace_value++ out = _sanitize_trace_value(+ {"outer": {"api_key": "secret", "text": "visible"}},+ )+ assert out["outer"]["api_key"] == "<redacted>"+ assert out["outer"]["text"] == "visible"diff --git a/tests/messaging/test_handler.py b/tests/messaging/test_handler.pyindex 2c79bbbf..6ea7380d 100644--- a/tests/messaging/test_handler.py+++ b/tests/messaging/test_handler.py@@ -14,10 +14,11 @@ def handler(mock_platform, mock_cli_manager, mock_session_store):@pytest.mark.asyncio-async def test_handle_message_default_logs_text_len_not_content(+async def test_handle_message_turn_trace_includes_full_message_text(mock_platform, mock_cli_manager, mock_session_store, incoming_message_factory):- secret = "user-secret-content-never-log-default"+ """turn.received always records the verbatim user message (local debugging)."""+ secret = "user-message-content-visible-in-trace"handler = ClaudeMessageHandler(mock_platform,mock_cli_manager,@@ -27,33 +28,33 @@ async def test_handle_message_default_logs_text_len_not_content(incoming = incoming_message_factory(text=secret)with (patch.object(handler, "_handle_message_impl", new_callable=AsyncMock),- patch("messaging.handler.logger.info") as log_info,+ patch("messaging.handler.trace_event") as trace_mock,):await handler.handle_message(incoming)- blob = " ".join(str(c) for c in log_info.call_args_list)- assert secret not in blob- assert "text_len=" in blob+ kwargs = trace_mock.call_args.kwargs+ assert kwargs["event"] == "turn.received"+ assert kwargs["message_text"] == secret@pytest.mark.asyncio-async def test_handle_message_raw_content_logging_includes_preview(+async def test_handle_message_log_raw_messaging_does_not_change_turn_received_shape(mock_platform, mock_cli_manager, mock_session_store, incoming_message_factory):- secret = "visible-preview-xyz"+ """LOG_RAW_MESSAGING_CONTENT is adapter-only; ingress TRACE always includes text."""+ text = "visible-either-way"handler = ClaudeMessageHandler(mock_platform,mock_cli_manager,mock_session_store,log_raw_messaging_content=True,)- incoming = incoming_message_factory(text=secret)+ incoming = incoming_message_factory(text=text)with (patch.object(handler, "_handle_message_impl", new_callable=AsyncMock),- patch("messaging.handler.logger.info") as log_info,+ patch("messaging.handler.trace_event") as trace_mock,):await handler.handle_message(incoming)- blob = " ".join(str(c) for c in log_info.call_args_list)- assert secret in blob+ assert trace_mock.call_args.kwargs["message_text"] == textdef test_get_initial_status_new_conversation(handler):