Skip to content

Your lightweight open-source personal agent.

fix: improve log reliability and correlation

chengyongru8a7ca8a65476
diff --git a/nanobot/agent/loop.py b/nanobot/agent/loop.py
index 4fc185213..7d8cec93c 100644
--- a/nanobot/agent/loop.py
+++ b/nanobot/agent/loop.py
@@ -1566,10 +1566,9 @@ class AgentLoop:
try:
await delivery.abort_stream()
except Exception:
- logger.debug(
+ logger.opt(exception=log_content).debug(
"Could not close stream for cancelled session {}",
session_key,
- exc_info=True,
)
# An explicit turn stop materializes partial context so
# the next prompt can see completed tool results. Gateway
@@ -1591,10 +1590,9 @@ class AgentLoop:
key,
)
except Exception:
- logger.debug(
+ logger.opt(exception=log_content).debug(
"Could not restore checkpoint for cancelled session {}",
session_key,
- exc_info=True,
)
raise
except Exception as exc:
@@ -1779,15 +1777,18 @@ class AgentLoop:
ctx.events = EventSink(track_output, ctx.events.accepts)
- await self._run_turn_stage(ctx, "restore", self._restore_turn)
- await self._run_turn_stage(ctx, "compact", self._compact_session)
- if await self._run_turn_stage(ctx, "command", self._dispatch_command):
+ with logger.contextualize(turn_id=ctx.turn_id, session_key=ctx.session_key):
+ await self._run_turn_stage(ctx, "restore", self._restore_turn)
+ await self._run_turn_stage(ctx, "compact", self._compact_session)
+ if await self._run_turn_stage(ctx, "command", self._dispatch_command):
+ self._log_turn_completion(ctx, outcome="command")
+ return ctx.outbound
+ await self._run_turn_stage(ctx, "build", self._build_turn)
+ await self._run_turn_stage(ctx, "run", self._run_turn)
+ await self._run_turn_stage(ctx, "save", self._persist_turn)
+ await self._run_turn_stage(ctx, "respond", self._prepare_outbound)
+ self._log_turn_completion(ctx)
return ctx.outbound
- await self._run_turn_stage(ctx, "build", self._build_turn)
- await self._run_turn_stage(ctx, "run", self._run_turn)
- await self._run_turn_stage(ctx, "save", self._persist_turn)
- await self._run_turn_stage(ctx, "respond", self._prepare_outbound)
- return ctx.outbound
async def _run_turn_stage(
self,
@@ -1800,22 +1801,76 @@ class AgentLoop:
result = await handler(ctx)
except Exception:
duration_ms = (time.perf_counter() - started_at) * 1000
- logger.debug(
- "[turn {}] Stage {} failed after {:.1f}ms",
- ctx.turn_id,
+ log_content = not ctx.ephemeral and (
+ ctx.session is None
+ or (ctx.session.policy.persist and ctx.session.policy.log_content)
+ )
+ logger.opt(exception=log_content).bind(
+ event="turn_stage",
+ stage=name,
+ outcome="error",
+ duration_ms=round(duration_ms, 1),
+ ).error(
+ "Stage {} failed after {:.1f}ms",
name,
duration_ms,
)
raise
duration_ms = (time.perf_counter() - started_at) * 1000
- logger.debug(
- "[turn {}] Stage {} completed in {:.1f}ms",
- ctx.turn_id,
+ logger.bind(
+ event="turn_stage",
+ stage=name,
+ outcome="success",
+ duration_ms=round(duration_ms, 1),
+ ).debug(
+ "Stage {} completed in {:.1f}ms",
name,
duration_ms,
)
return result
+ def _log_turn_completion(self, ctx: TurnContext, *, outcome: str | None = None) -> None:
+ duration_ms = ctx.turn_latency_ms
+ if duration_ms is None:
+ duration_ms = max(0, round((time.time() - ctx.turn_wall_started_at) * 1000))
+ runtime = ctx.runtime
+ provider = runtime.provider.provider_name if runtime is not None else None
+ model = runtime.model if runtime is not None else None
+ response_preview = "[content hidden]"
+ if (
+ ctx.kind is TurnKind.USER
+ and ctx.session is not None
+ and not ctx.ephemeral
+ and ctx.session.policy.persist
+ and ctx.session.policy.log_content
+ and ctx.final_content is not None
+ ):
+ response_preview = (
+ f"{ctx.final_content[:120]}..."
+ if len(ctx.final_content) > 120
+ else ctx.final_content
+ )
+ final_outcome = outcome or ctx.stop_reason or "completed"
+ logger.bind(
+ event="turn_completed",
+ outcome=final_outcome,
+ duration_ms=duration_ms,
+ provider=provider,
+ model=model,
+ channel=ctx.msg.channel,
+ chat_id=ctx.msg.chat_id,
+ ).info(
+ "Turn completed channel={} chat_id={} outcome={} duration_ms={} "
+ "provider={} model={} response={}",
+ ctx.msg.channel,
+ ctx.msg.chat_id,
+ final_outcome,
+ duration_ms,
+ provider or "-",
+ model or "-",
+ response_preview,
+ )
+
def _assemble_outbound(
self,
msg: InboundMessage,
@@ -1823,16 +1878,9 @@ class AgentLoop:
stop_reason: str,
streamed_content: bool,
*,
- log_content: bool = True,
turn_latency_ms: int | None = None,
) -> OutboundMessage | None:
"""Assemble the final outbound message from turn results."""
- if log_content:
- preview = final_content[:120] + "..." if len(final_content) > 120 else final_content
- logger.info("Response to {}:{}: {}", msg.channel, msg.sender_id, preview)
- else:
- logger.info("Response to {}:{}: [content hidden]", msg.channel, msg.sender_id)
-
event = None
meta = dict(msg.metadata or {})
if streamed_content and stop_reason not in {"error", "tool_error"}:
@@ -2184,7 +2232,6 @@ class AgentLoop:
cast(str, ctx.final_content),
ctx.stop_reason,
ctx.streamed_content,
- log_content=ctx.require_session().policy.log_content,
turn_latency_ms=ctx.turn_latency_ms,
)
if ctx.ephemeral and ctx.outbound is not None:
diff --git a/nanobot/api/server.py b/nanobot/api/server.py
index ac7709572..a41b5c436 100644
--- a/nanobot/api/server.py
+++ b/nanobot/api/server.py
@@ -51,6 +51,7 @@ _MODEL_NAME_KEY = web.AppKey[str]("model_name")
_REQUEST_TIMEOUT_KEY = web.AppKey[float]("request_timeout")
_SESSION_LOCKS_KEY = web.AppKey[dict[str, asyncio.Lock]]("session_locks")
_PREPARE_AGENT_KEY = web.AppKey[Callable[[], Awaitable[None]] | None]("prepare_agent")
+_REQUEST_ID_KEY = web.RequestKey[str]("request_id")
_MISSING = object()
@@ -498,6 +499,60 @@ def create_app(
app[_SESSION_LOCKS_KEY] = {} # per-user locks, keyed by session_key
app[_PREPARE_AGENT_KEY] = prepare_agent
+ @web.middleware
+ async def request_logging_middleware(
+ request: web.Request,
+ handler: Callable[[web.Request], Awaitable[web.StreamResponse]],
+ ) -> web.StreamResponse:
+ request_id = uuid.uuid4().hex[:16]
+ request[_REQUEST_ID_KEY] = request_id
+ started_at = time.perf_counter()
+ with logger.contextualize(request_id=request_id):
+ try:
+ response = await handler(request)
+ except Exception:
+ duration_ms = round((time.perf_counter() - started_at) * 1000, 1)
+ logger.opt(exception=True).bind(
+ event="http_request",
+ outcome="error",
+ duration_ms=duration_ms,
+ method=request.method,
+ path=request.path,
+ ).error(
+ "HTTP request failed method={} path={} duration_ms={}",
+ request.method,
+ request.path,
+ duration_ms,
+ )
+ raise
+
+ duration_ms = round((time.perf_counter() - started_at) * 1000, 1)
+ request_log = logger.bind(
+ event="http_request",
+ outcome="success" if response.status < 400 else "error",
+ duration_ms=duration_ms,
+ method=request.method,
+ path=request.path,
+ status_code=response.status,
+ )
+ log_method = request_log.debug if request.path == "/health" else request_log.info
+ log_method(
+ "HTTP request completed method={} path={} status={} duration_ms={}",
+ request.method,
+ request.path,
+ response.status,
+ duration_ms,
+ )
+ return response
+
+ async def add_request_id_header(
+ request: web.Request,
+ response: web.StreamResponse,
+ ) -> None:
+ request_id = request.get(_REQUEST_ID_KEY)
+ if request_id:
+ response.headers["X-Request-ID"] = request_id
+
@web.middleware
async def auth_middleware(
request: web.Request,
@@ -515,7 +570,8 @@ def create_app(
return _error_json(401, "Invalid API key")
return await handler(request)
- app.middlewares.append(auth_middleware)
+ app.middlewares.extend((request_logging_middleware, auth_middleware))
+ app.on_response_prepare.append(add_request_id_header)
app.router.add_post("/v1/chat/completions", handle_chat_completions)
app.router.add_get("/v1/models", handle_models)
diff --git a/nanobot/channels/dingtalk/runtime.py b/nanobot/channels/dingtalk/runtime.py
index e21899168..982398669 100644
--- a/nanobot/channels/dingtalk/runtime.py
+++ b/nanobot/channels/dingtalk/runtime.py
@@ -188,7 +188,9 @@ class NanobotDingTalkHandler(_CallbackHandlerBase):
)
return AckMessage.STATUS_OK, "OK"
- self.channel.logger.info("Received message from {} ({}): {}", sender_name, sender_id, content)
+ self.channel.logger.debug(
+ "Received DingTalk message from {} ({})", sender_name, sender_id
+ )
# Forward to Nanobot via _on_message (non-blocking).
# Store reference to prevent GC before task completes.
@@ -367,7 +369,7 @@ class DingTalkChannel(BaseChannel):
if isawaitable(result):
await result
except Exception:
- self.logger.debug("DingTalk stream client close failed", exc_info=True)
+ self.logger.opt(exception=True).debug("DingTalk stream client close failed")
async def _get_access_token(self) -> str | None:
"""Get or refresh Access Token."""
@@ -824,7 +826,6 @@ class DingTalkChannel(BaseChannel):
permission checks before publishing to the bus.
"""
try:
- self.logger.info("inbound: {} from {}", content, sender_name)
if not sender_id:
self.logger.warning("dropping DingTalk message without a sender ID")
return
diff --git a/nanobot/channels/matrix/runtime.py b/nanobot/channels/matrix/runtime.py
index bbcf246fd..a2c78a2b8 100644
--- a/nanobot/channels/matrix/runtime.py
+++ b/nanobot/channels/matrix/runtime.py
@@ -536,7 +536,7 @@ class MatrixChannel(BaseChannel):
try:
response = await self.client.content_repository_config()
except Exception:
- self.logger.error("Failed to fetch server upload limit", exc_info=True)
+ self.logger.opt(exception=True).error("Failed to fetch server upload limit")
return None
upload_size = getattr(response, "upload_size", None)
if isinstance(upload_size, int) and upload_size > 0:
@@ -582,7 +582,7 @@ class MatrixChannel(BaseChannel):
filesize=size_bytes,
)
except Exception:
- self.logger.error("Matrix media upload failed for {}", filename, exc_info=True)
+ self.logger.opt(exception=True).error("Matrix media upload failed for {}", filename)
return fail
is_tuple_result = isinstance(cast(object, upload_result), tuple)
@@ -607,7 +607,9 @@ class MatrixChannel(BaseChannel):
try:
await self._send_room_content(room_id, content)
except Exception:
- self.logger.error("Matrix room content send failed for room_id={}", room_id, exc_info=True)
+ self.logger.opt(exception=True).error(
+ "Matrix room content send failed for room_id={}", room_id
+ )
return fail
return None
@@ -712,7 +714,9 @@ class MatrixChannel(BaseChannel):
buf.text = previous_text
if created_buf:
self._stream_bufs.pop(stream_key, None)
- self.logger.error("Stream send/edit failed for chat_id={}", chat_id, exc_info=True)
+ self.logger.opt(exception=True).error(
+ "Stream send/edit failed for chat_id={}", chat_id
+ )
await self._stop_typing_keepalive(chat_id, clear_typing=True)
raise
@@ -1011,7 +1015,9 @@ class MatrixChannel(BaseChannel):
),
)
except Exception:
- self.logger.error("Matrix join request exception for room={}", room_id, exc_info=True)
+ self.logger.opt(exception=True).error(
+ "Matrix join request exception for room={}", room_id
+ )
return False
if isinstance(resp, JoinError):
self.logger.error("Matrix auto-join failed for room={}: {}", room_id, resp)
@@ -1287,7 +1293,7 @@ class MatrixChannel(BaseChannel):
except _MediaTooLargeError:
raise
except (aiohttp.ClientError, asyncio.TimeoutError, OSError):
- self.logger.warning("download failed for {}", mxc_url, exc_info=True)
+ self.logger.opt(exception=True).warning("download failed for {}", mxc_url)
return None
def _decrypt_media_bytes(self, event: MatrixMediaEvent, ciphertext: bytes) -> bytes | None:
diff --git a/nanobot/channels/matrix/tests/test_matrix_channel.py b/nanobot/channels/matrix/tests/test_matrix_channel.py
index c9e961ae5..967c99d62 100644
--- a/nanobot/channels/matrix/tests/test_matrix_channel.py
+++ b/nanobot/channels/matrix/tests/test_matrix_channel.py
@@ -1838,8 +1838,9 @@ async def test_send_handles_upload_exception_and_reports_failure(tmp_path) -> No
client.room_send_calls[0]["content"]["body"]
== "Please review.\n[attachment: broken.txt - upload failed]"
)
- channel.logger.error.assert_called_once_with(
- "Matrix media upload failed for {}", "broken.txt", exc_info=True
+ channel.logger.opt.assert_called_once_with(exception=True)
+ channel.logger.opt.return_value.error.assert_called_once_with(
+ "Matrix media upload failed for {}", "broken.txt"
)
@@ -1861,10 +1862,10 @@ async def test_attachment_room_send_error_logs_room_id(tmp_path) -> None:
)
assert failure == "[attachment: report.txt - upload failed]"
- channel.logger.error.assert_called_once_with(
+ channel.logger.opt.assert_called_once_with(exception=True)
+ channel.logger.opt.return_value.error.assert_called_once_with(
"Matrix room content send failed for room_id={}",
"!room:matrix.org",
- exc_info=True,
)
@@ -2509,8 +2510,9 @@ async def test_send_delta_on_error_restores_buffer_and_raises(monkeypatch) -> No
assert len(client.room_send_calls) == 1
assert len(client.typing_calls) == 1
- channel.logger.error.assert_called_once_with(
- "Stream send/edit failed for chat_id={}", "!room:matrix.org", exc_info=True
+ channel.logger.opt.assert_called_once_with(exception=True)
+ channel.logger.opt.return_value.error.assert_called_once_with(
+ "Stream send/edit failed for chat_id={}", "!room:matrix.org"
)
client.raise_on_send = False
diff --git a/nanobot/channels/signal/runtime.py b/nanobot/channels/signal/runtime.py
index 1235ce5fe..f00efe0ae 100644
--- a/nanobot/channels/signal/runtime.py
+++ b/nanobot/channels/signal/runtime.py
@@ -624,10 +624,6 @@ class SignalChannel(BaseChannel):
if not self._running:
break
- # Debug: log raw SSE lines (except keepalive pings)
- if line and line != ":":
- self.logger.debug("SSE line received: {}", line[:200])
-
# SSE format handling
if isinstance(line, str): # pyright: ignore[reportUnnecessaryIsInstance]
# Empty line signals end of event
@@ -641,7 +637,6 @@ class SignalChannel(BaseChannel):
if data is None:
self.logger.warning("Ignoring non-object SSE event: {}", data_str[:200])
continue
- self.logger.debug("SSE event parsed: {}", data)
await self._handle_receive_notification(data)
except json.JSONDecodeError as e:
self.logger.warning(
@@ -690,13 +685,10 @@ class SignalChannel(BaseChannel):
async def _handle_receive_notification(self, params: dict[str, Any]) -> None:
"""Handle incoming message notification from signal-cli."""
- self.logger.debug("_handle_receive_notification called with: {}", params)
async with self._safe_handle("receive notification", params):
# Extract envelope from SSE notification: {"envelope": {...}}
envelope = _as_json_object(params.get("envelope"))
- self.logger.debug("Extracted envelope: {}", envelope)
-
if envelope is None:
self.logger.debug("No envelope found in params")
return
@@ -814,7 +806,7 @@ class SignalChannel(BaseChannel):
chat_id=chat_id,
)
- self.logger.debug("Signal message from {}: {}...", sender_number, content[:50])
+ self.logger.debug("Received Signal message from {}", sender_number)
await self._start_typing(chat_id)
try:
diff --git a/nanobot/channels/weixin/runtime.py b/nanobot/channels/weixin/runtime.py
index 018e9bb90..ff1449c4b 100644
--- a/nanobot/channels/weixin/runtime.py
+++ b/nanobot/channels/weixin/runtime.py
@@ -423,7 +423,7 @@ class WeixinChannel(BaseChannel):
self.config.base_url = base_url
return bool(self._token)
except Exception:
- self.logger.error("Failed to load Weixin account state", exc_info=True)
+ self.logger.opt(exception=True).error("Failed to load Weixin account state")
return False
def _save_state(self, *, force: bool = False) -> None:
diff --git a/nanobot/cli/commands.py b/nanobot/cli/commands.py
index cb655415f..49f7c716e 100644
--- a/nanobot/cli/commands.py
+++ b/nanobot/cli/commands.py
@@ -25,20 +25,10 @@ import typer # noqa: E402
from loguru import logger # noqa: E402
from typer.core import TyperGroup # noqa: E402
+from nanobot.utils.log_config import configure_console_logging # noqa: E402
+
# Remove default handler and re-add with unified nanobot format
-logger.remove()
-_log_handler_id = logger.add(
- sys.stderr,
- format=(
- "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
- "<level>{level: <5}</level> | "
- "<cyan>{extra[channel]}</cyan> | "
- "<level>{message}</level>"
- ),
- level="INFO",
- colorize=None,
- filter=lambda record: record["extra"].setdefault("channel", "-") or True,
-)
+_log_handler_id = configure_console_logging(sys.stderr)
from rich.console import Console # noqa: E402
diff --git a/nanobot/cli/entry.py b/nanobot/cli/entry.py
index 972a25b23..30fb37b87 100644
--- a/nanobot/cli/entry.py
+++ b/nanobot/cli/entry.py
@@ -7,6 +7,8 @@ import sys
from contextlib import suppress
from nanobot.cli.process_identity import set_cli_process_identity
+from nanobot.utils.log_config import configure_console_logging
+from nanobot.utils.rotating_output import configure_background_output_from_env
_ROOT_OPTIONS = frozenset(
{
@@ -58,6 +60,7 @@ def _configure_windows_console() -> None:
def _run_agent(args: list[str], *, prog_name: str) -> None:
"""Run the shared agent command without importing the complete CLI graph."""
+ configure_console_logging(sys.stderr)
import typer
from nanobot.cli.agent import agent
@@ -70,6 +73,7 @@ def _run_agent(args: list[str], *, prog_name: str) -> None:
def main() -> None:
"""Dispatch native TUI startup without importing the complete CLI graph."""
+ configure_background_output_from_env()
raw_args = sys.argv[1:]
# Installed completion scripts call ``nanobot`` without positional arguments
# and pass the request through this environment variable. Keep those requests
diff --git a/nanobot/cli/gateway.py b/nanobot/cli/gateway.py
index d4a668745..1ec025cb6 100644
--- a/nanobot/cli/gateway.py
+++ b/nanobot/cli/gateway.py
@@ -26,6 +26,7 @@ from nanobot.gateway.service import (
GatewayServiceResult,
ServiceManagerKind,
)
+from nanobot.utils.log_config import add_console_log_sink
from nanobot.webui.build import BuildMode
RuntimeConfigLoader = Callable[[str | None, str | None], Config]
@@ -66,18 +67,7 @@ def create_gateway_app(
if not verbose:
return
logger.remove(log_handler_id)
- logger.add(
- sys.stderr,
- format=(
- "<green>{time:YYYY-MM-DD HH:mm:ss}</green> | "
- "<level>{level: <5}</level> | "
- "<cyan>{extra[channel]}</cyan> | "
- "<level>{message}</level>"
- ),
- level="DEBUG",
- colorize=None,
- filter=lambda record: record["extra"].setdefault("channel", "-") or True,
- )
+ add_console_log_sink(sys.stderr, level="DEBUG")
def instance_for_selectors(
*,
diff --git a/nanobot/process_runtime.py b/nanobot/process_runtime.py
index c5e425511..93bad4cbc 100644
--- a/nanobot/process_runtime.py
+++ b/nanobot/process_runtime.py
@@ -23,6 +23,14 @@ from typing import Any, Generic, Literal, TypeVar, cast
from filelock import FileLock
+from nanobot.utils.rotating_output import (
+ BACKGROUND_LOG_BACKUP_COUNT_ENV,
+ BACKGROUND_LOG_MAX_BYTES_ENV,
+ BACKGROUND_LOG_PATH_ENV,
+ DEFAULT_BACKUP_COUNT,
+ DEFAULT_MAX_BYTES,
+)
+
@dataclass(frozen=True)
class ProcessStartOptions:
@@ -110,12 +118,17 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
self.paths.run_dir.mkdir(parents=True, exist_ok=True)
self.paths.logs_dir.mkdir(parents=True, exist_ok=True)
+ child_env = os.environ.copy()
+ child_env[BACKGROUND_LOG_PATH_ENV] = str(self.paths.log_path)
+ child_env[BACKGROUND_LOG_MAX_BYTES_ENV] = str(DEFAULT_MAX_BYTES)
+ child_env[BACKGROUND_LOG_BACKUP_COUNT_ENV] = str(DEFAULT_BACKUP_COUNT)
with self.paths.log_path.open("a", encoding="utf-8") as log_handle:
process = self._popen(
command,
stdin=subprocess.DEVNULL,
stdout=log_handle,
stderr=subprocess.STDOUT,
+ env=child_env,
**self._popen_platform_kwargs(),
)
self._owned_process = process
@@ -229,10 +242,22 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
"""Return the last ``tail`` log lines."""
if tail <= 0 or not self.paths.log_path.exists():
return []
- try:
- lines = self.paths.log_path.read_text(encoding="utf-8", errors="replace").splitlines()
- except OSError:
- return []
+ lines: list[str] = []
+ paths = [
+ self.paths.log_path,
+ *(
+ self.paths.log_path.with_name(f"{self.paths.log_path.name}.{index}")
+ for index in range(1, DEFAULT_BACKUP_COUNT + 1)
+ ),
+ ]
+ for path in paths:
+ try:
+ older_lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
+ except OSError:
+ continue
+ lines = older_lines + lines
+ if len(lines) >= tail:
+ break
return lines[-tail:]
def follow_logs(self, *, tail: int = 200) -> int:
@@ -241,17 +266,35 @@ class ManagedProcessRuntime(Generic[_StartOptionsT]):
print(line)
self.paths.logs_dir.mkdir(parents=True, exist_ok=True)
self.paths.log_path.touch(exist_ok=True)
+ handle = self.paths.log_path.open("r", encoding="utf-8", errors="replace")
try:
- with self.paths.log_path.open("r", encoding="utf-8", errors="replace") as handle:
+ try:
handle.seek(0, os.SEEK_END)
while True:
line = handle.readline()
if line:
print(line.rstrip("\n"))
+ elif self._log_file_replaced(handle):
+ handle.close()
+ handle = self.paths.log_path.open(
+ "r", encoding="utf-8", errors="replace"
+ )
else:
self._sleep(0.5)
- except KeyboardInterrupt:
- return 130
+ except KeyboardInterrupt:
+ return 130
+ finally:
+ handle.close()
+
+ def _log_file_replaced(self, handle: Any) -> bool:
+ """Return whether rotation replaced or truncated the followed log file."""
+ try:
+ opened = os.fstat(handle.fileno())
+ current = self.paths.log_path.stat()
+ same_file = (opened.st_dev, opened.st_ino) == (current.st_dev, current.st_ino)
+ return not same_file or current.st_size < handle.tell()
+ except OSError:
+ return False
def process_identity(self, pid: int) -> str | int | None:
"""Return an identity that changes when an operating-system PID is reused."""
diff --git a/nanobot/session/manager.py b/nanobot/session/manager.py
index 770ce3724..a4035fd99 100644
--- a/nanobot/session/manager.py
+++ b/nanobot/session/manager.py
@@ -1750,7 +1750,7 @@ class SessionManager:
self.save(session, fsync=True)
flushed += 1
except Exception:
- logger.warning("Failed to flush session {}", key, exc_info=True)
+ logger.opt(exception=True).warning("Failed to flush session {}", key)
return flushed
def invalidate(self, key: str) -> None:
diff --git a/nanobot/session/webui_turns.py b/nanobot/session/webui_turns.py
index 5baecd1a3..418181e37 100644
--- a/nanobot/session/webui_turns.py
+++ b/nanobot/session/webui_turns.py
@@ -275,7 +275,9 @@ async def maybe_generate_webui_title(
retry_mode="standard",
)
except Exception:
- logger.debug("Failed to generate webui session title for {}", session_key, exc_info=True)
+ logger.opt(exception=True).debug(
+ "Failed to generate webui session title for {}", session_key
+ )
return False
title = clean_generated_title(response.content)
@@ -650,10 +652,9 @@ class WebuiTurnCoordinator:
session_message=public_metadata,
)
except (OSError, TypeError, ValueError):
- logger.warning(
+ logger.opt(exception=True).warning(
"Failed to persist session input {}",
envelope["message_id"],
- exc_info=True,
)
await self.bus.publish_outbound(outbound_message_for_event(
channel="websocket",
diff --git a/nanobot/utils/log_config.py b/nanobot/utils/log_config.py
new file mode 100644
index 000000000..307f915b6
--- /dev/null
+++ b/nanobot/utils/log_config.py
@@ -0,0 +1,59 @@
+"""Shared configuration for human-readable nanobot logs."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from loguru import logger
+
+if TYPE_CHECKING:
+ from loguru import Record
+
+LOG_FORMAT = (
+ "<green>{time:YYYY-MM-DDTHH:mm:ss.SSSZZ}</green> | "
+ "<level>{level: <7}</level> | "
+ "<cyan>{extra[channel]}</cyan> | "
+ "<cyan>{extra[log_context]}</cyan> | "
+ "<level>{extra[safe_message]}</level>"
+)
+
+
+def _single_line(value: object) -> str:
+ return str(value).replace("\r", "\\r").replace("\n", "\\n")
+
+
+def prepare_log_record(record: "Record") -> bool:
+ """Populate display fields without altering the structured log record."""
+ extra = record["extra"]
+ extra.setdefault("channel", "-")
+ extra["channel"] = _single_line(extra["channel"])
+ extra["safe_message"] = _single_line(record["message"])
+
+ context: list[str] = []
+ for label, field in (
+ ("request", "request_id"),
+ ("turn", "turn_id"),
+ ("session", "session_key"),
+ ):
+ value = extra.get(field)
+ if value not in (None, "", "-"):
+ context.append(f"{label}={_single_line(value)}")
+ extra["log_context"] = " ".join(context) or "-"
+ return True
+
+
+def add_console_log_sink(stream: Any, *, level: str = "INFO") -> int:
+ """Add the standard nanobot console sink and return its handler id."""
+ return logger.add(
+ stream,
+ format=LOG_FORMAT,
+ level=level,
+ colorize=None,
+ filter=prepare_log_record,
+ )
+
+
+def configure_console_logging(stream: Any, *, level: str = "INFO") -> int:
+ """Replace existing handlers with the standard nanobot console sink."""
+ logger.remove()
+ return add_console_log_sink(stream, level=level)
diff --git a/nanobot/utils/rotating_output.py b/nanobot/utils/rotating_output.py
new file mode 100644
index 000000000..b45c069d4
--- /dev/null
+++ b/nanobot/utils/rotating_output.py
@@ -0,0 +1,168 @@
+"""Bounded text output for detached nanobot processes."""
+
+from __future__ import annotations
+
+import io
+import os
+import shutil
+import sys
+import threading
+from contextlib import suppress
+from pathlib import Path
+from typing import BinaryIO
+
+BACKGROUND_LOG_PATH_ENV = "_NANOBOT_BACKGROUND_LOG_PATH"
+BACKGROUND_LOG_MAX_BYTES_ENV = "_NANOBOT_BACKGROUND_LOG_MAX_BYTES"
+BACKGROUND_LOG_BACKUP_COUNT_ENV = "_NANOBOT_BACKGROUND_LOG_BACKUP_COUNT"
+
+DEFAULT_MAX_BYTES = 10 * 1024 * 1024
+DEFAULT_BACKUP_COUNT = 5
+
+
+class RotatingTextOutput(io.TextIOBase):
+ """A UTF-8 text stream that rotates a file before it exceeds its size cap."""
+
+ def __init__(self, path: Path, *, max_bytes: int, backup_count: int) -> None:
+ super().__init__()
+ self._path = path
+ self._max_bytes = max(1, max_bytes)
+ self._backup_count = max(0, backup_count)
+ self._handle: BinaryIO | None = None
+ self._size = 0
+ self._lock = threading.RLock()
+
+ @property
+ def encoding(self) -> str: # pyright: ignore[reportIncompatibleVariableOverride]
+ return "utf-8"
+
+ @property
+ def errors(self) -> str: # pyright: ignore[reportIncompatibleVariableOverride]
+ return "replace"
+
+ def writable(self) -> bool:
+ return True
+
+ def isatty(self) -> bool:
+ return False
+
+ def fileno(self) -> int:
+ with self._lock:
+ self._ensure_open()
+ assert self._handle is not None
+ return self._handle.fileno()
+
+ def write(self, text: str) -> int:
+ if not text:
+ return 0
+ data = text.encode(self.encoding, errors=self.errors)
+ with self._lock:
+ self._ensure_open()
+ if self._size and self._size + len(data) > self._max_bytes:
+ self._rotate()
+ self._ensure_open()
+ assert self._handle is not None
+ self._handle.write(data)
+ self._handle.flush()
+ self._size += len(data)
+ return len(text)
+
+ def flush(self) -> None:
+ with self._lock:
+ if self._handle is not None:
+ self._handle.flush()
+
+ def close(self) -> None:
+ with self._lock:
+ if self._handle is not None:
+ self._handle.close()
+ self._handle = None
+ super().close()
+
+ def _ensure_open(self) -> None:
+ if self._handle is not None:
+ return
+ self._path.parent.mkdir(parents=True, exist_ok=True)
+ size = self._path.stat().st_size if self._path.exists() else 0
+ if size >= self._max_bytes:
+ self._rotate()
+ size = self._path.stat().st_size if self._path.exists() else 0
+ self._handle = self._path.open("ab")
+ self._size = size
+
+ def _rotate(self) -> None:
+ if self._handle is not None:
+ self._handle.flush()
+ self._handle.close()
+ self._handle = None
+
+ if self._backup_count == 0:
+ try:
+ self._path.unlink(missing_ok=True)
+ self._size = 0
+ except OSError:
+ self._size = self._path.stat().st_size if self._path.exists() else 0
+ return
+
+ try:
+ oldest = self._backup_path(self._backup_count)
+ oldest.unlink(missing_ok=True)
+ for index in range(self._backup_count - 1, 0, -1):
+ source = self._backup_path(index)
+ if source.exists():
+ source.replace(self._backup_path(index + 1))
+ if self._path.exists():
+ try:
+ self._path.replace(self._backup_path(1))
+ except PermissionError:
+ # A Windows ``gateway logs --follow`` reader can prevent rename.
+ # Copy and truncate preserves both retention and the live stream.
+ shutil.copyfile(self._path, self._backup_path(1))
+ self._path.open("wb").close()
+ self._size = 0
+ except OSError:
+ # Logging must remain available even when rotation is temporarily blocked.
+ self._size = self._path.stat().st_size if self._path.exists() else 0
+
+ def _backup_path(self, index: int) -> Path:
+ return self._path.with_name(f"{self._path.name}.{index}")
+
+
+def _positive_int(value: str | None, default: int, *, minimum: int) -> int:
+ try:
+ return max(minimum, int(value or ""))
+ except ValueError:
+ return default
+
+
+def configure_background_output_from_env() -> RotatingTextOutput | None:
+ """Replace inherited output handles when launched by ``ManagedProcessRuntime``."""
+ raw_path = os.environ.pop(BACKGROUND_LOG_PATH_ENV, "")
+ if not raw_path:
+ return None
+ max_bytes = _positive_int(
+ os.environ.pop(BACKGROUND_LOG_MAX_BYTES_ENV, None),
+ DEFAULT_MAX_BYTES,
+ minimum=1,
+ )
+ backup_count = _positive_int(
+ os.environ.pop(BACKGROUND_LOG_BACKUP_COUNT_ENV, None),
+ DEFAULT_BACKUP_COUNT,
+ minimum=0,
+ )
+ stream = RotatingTextOutput(
+ Path(raw_path),
+ max_bytes=max_bytes,
+ backup_count=backup_count,
+ )
+
+ replaced = {id(output): output for output in (sys.stdout, sys.stderr)}
+ sys.stdout = stream
+ sys.stderr = stream
+ sys.__stdout__ = stream
+ sys.__stderr__ = stream
+ for output in replaced.values():
+ with suppress(Exception):
+ output.flush()
+ with suppress(Exception):
+ output.close()
+ return stream
diff --git a/tests/agent/test_loop_save_turn.py b/tests/agent/test_loop_save_turn.py
index 384e5e461..fdb4fbeb4 100644
--- a/tests/agent/test_loop_save_turn.py
+++ b/tests/agent/test_loop_save_turn.py
@@ -121,6 +121,7 @@ def _runtime_message(content, blocks: list[RuntimeContextBlock]) -> dict:
def _make_full_loop(tmp_path: Path) -> AgentLoop:
provider = MagicMock()
+ provider.provider_name = "test"
provider.get_default_model.return_value = "test-model"
provider.generation = SimpleNamespace(max_tokens=4096)
provider.chat_stream_with_retry = AsyncMock(return_value=LLMResponse(content="Test title"))
@@ -2105,8 +2106,8 @@ async def test_system_subagent_followup_uses_common_turn_lifecycle(tmp_path: Pat
loop._run_agent_loop = fake_run_agent_loop # type: ignore[method-assign]
- logs: list[str] = []
- sink_id = logger.add(logs.append, level="DEBUG", format="{message}")
+ records = []
+ sink_id = logger.add(lambda message: records.append(message.record), level="DEBUG")
try:
await loop._process_message(
InboundMessage(
@@ -2129,9 +2130,64 @@ async def test_system_subagent_followup_uses_common_turn_lifecycle(tmp_path: Pat
"_persist_turn",
"_prepare_outbound",
]
- logged = "".join(logs)
+ logged = "\n".join(record["message"] for record in records)
for stage in ("restore", "compact", "command", "build", "run", "save", "respond"):
assert f"Stage {stage} completed in" in logged
+ stage_records = [record for record in records if record["extra"].get("event") == "turn_stage"]
+ assert {record["extra"]["stage"] for record in stage_records} == {
+ "restore",
+ "compact",
+ "command",
+ "build",
+ "run",
+ "save",
+ "respond",
+ }
+ assert {record["extra"]["session_key"] for record in stage_records} == {"cli:test"}
+ assert len({record["extra"]["turn_id"] for record in stage_records}) == 1
+ completion = next(
+ record for record in records if record["extra"].get("event") == "turn_completed"
+ )
+ assert completion["extra"]["outcome"] == "stop"
+ assert completion["extra"]["duration_ms"] >= 0
+ assert completion["extra"]["provider"] == "test"
+ assert completion["extra"]["model"] == "test-model"
+ assert "response=[content hidden]" in completion["message"]
+
+
+async def test_failed_turn_stage_logs_exception_and_correlation(tmp_path: Path) -> None:
+ loop = _make_full_loop(tmp_path)
+
+ async def fail_restore(_ctx) -> None:
+ raise RuntimeError("restore failed")
+
+ loop._restore_turn = fail_restore # type: ignore[method-assign]
+ records = []
+ sink_id = logger.add(lambda message: records.append(message.record), level="ERROR")
+ try:
+ with pytest.raises(RuntimeError, match="restore failed"):
+ await loop._process_message(
+ InboundMessage(
+ channel="cli",
+ sender_id="user",
+ chat_id="failure",
+ content="hello",
+ )
+ )
+ finally:
+ logger.remove(sink_id)
+
+ failure = next(
+ record
+ for record in records
+ if record["extra"].get("event") == "turn_stage"
+ and record["extra"].get("outcome") == "error"
+ )
+ assert failure["exception"] is not None
+ assert failure["extra"]["stage"] == "restore"
+ assert failure["extra"]["session_key"] == "cli:failure"
+ assert failure["extra"]["turn_id"].startswith("cli:failure:")
@pytest.mark.asyncio
diff --git a/tests/gateway/test_runtime.py b/tests/gateway/test_runtime.py
index 8dff4c0cb..dd5e981bb 100644
--- a/tests/gateway/test_runtime.py
+++ b/tests/gateway/test_runtime.py
@@ -22,6 +22,13 @@ from nanobot.gateway import (
)
from nanobot.gateway.runtime import monitor_gateway_clients
from nanobot.process_runtime import process_is_running
+from nanobot.utils.rotating_output import (
+ BACKGROUND_LOG_BACKUP_COUNT_ENV,
+ BACKGROUND_LOG_MAX_BYTES_ENV,
+ BACKGROUND_LOG_PATH_ENV,
+ DEFAULT_BACKUP_COUNT,
+ DEFAULT_MAX_BYTES,
+)
class FakeProcess:
@@ -234,6 +241,10 @@ def test_start_background_writes_state_and_child_command(tmp_path, monkeypatch):
"/tmp/config.json",
]
assert calls[0]["kwargs"]["start_new_session"] is True
+ child_env = calls[0]["kwargs"]["env"]
+ assert child_env[BACKGROUND_LOG_PATH_ENV] == str(runtime.paths.log_path)
+ assert child_env[BACKGROUND_LOG_MAX_BYTES_ENV] == str(DEFAULT_MAX_BYTES)
+ assert child_env[BACKGROUND_LOG_BACKUP_COUNT_ENV] == str(DEFAULT_BACKUP_COUNT)
state = json.loads(runtime.paths.state_path.read_text(encoding="utf-8"))
assert state["pid"] == 12345
assert state["identity"] == 12345
@@ -242,6 +253,26 @@ def test_start_background_writes_state_and_child_command(tmp_path, monkeypatch):
assert result.status.launch_mode == "background"
+def test_read_log_tail_spans_rotated_files(tmp_path: Path) -> None:
+ runtime = GatewayRuntime(paths=_paths(tmp_path))
+ runtime.paths.logs_dir.mkdir(parents=True)
+ runtime.paths.log_path.write_text("current-1\ncurrent-2\n", encoding="utf-8")
+ runtime.paths.log_path.with_name("gateway.log.1").write_text(
+ "backup-1\nbackup-2\n", encoding="utf-8"
+ )
+ runtime.paths.log_path.with_name("gateway.log.2").write_text(
+ "oldest\n", encoding="utf-8"
+ )
+
+ assert runtime.read_log_tail(tail=5) == [
+ "oldest",
+ "backup-1",
+ "backup-2",
+ "current-1",
+ "current-2",
+ ]
+
+
def test_foreground_gateway_claim_is_discoverable_and_released(tmp_path, monkeypatch):
runtime = GatewayRuntime(
paths=_paths(tmp_path),
diff --git a/tests/test_api_stream.py b/tests/test_api_stream.py
index a3d77d32d..3e267b764 100644
--- a/tests/test_api_stream.py
+++ b/tests/test_api_stream.py
@@ -112,6 +112,7 @@ async def test_stream_true_returns_sse(aiohttp_client) -> None:
)
assert resp.status == 200
assert resp.content_type == "text/event-stream"
+ assert resp.headers["X-Request-ID"]
body = await resp.text()
lines = [line for line in body.split("\n") if line.startswith("data: ")]
diff --git a/tests/test_openai_api.py b/tests/test_openai_api.py
index 86a1a04e2..57b752b0c 100644
--- a/tests/test_openai_api.py
+++ b/tests/test_openai_api.py
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
import pytest_asyncio
+from loguru import logger
from nanobot.agent.hook import AgentHook, AgentRunHookContext
from nanobot.api.server import (
@@ -126,10 +127,35 @@ async def test_api_key_protects_api_routes_but_not_health(aiohttp_client, mock_a
assert missing.status == 401
assert wrong.status == 401
assert ok.status == 200
+ assert health.headers["X-Request-ID"]
+ assert ok.headers["X-Request-ID"]
+ assert health.headers["X-Request-ID"] != ok.headers["X-Request-ID"]
assert (await missing.json())["error"]["message"].startswith("Missing Authorization")
assert (await wrong.json())["error"]["message"] == "Invalid API key"
[email protected](not HAS_AIOHTTP, reason="aiohttp not installed")
+async def test_api_request_log_has_correlation_and_completion_fields(
+ aiohttp_client, mock_agent
+) -> None:
+ records = []
+ sink = logger.add(lambda message: records.append(message.record), level="INFO")
+ try:
+ client = await aiohttp_client(create_app(mock_agent, api_key=API_KEY))
+ response = await client.get("/v1/models", headers=AUTH_HEADERS)
+ finally:
+ logger.remove(sink)
+
+ completion = next(
+ record for record in records if record["extra"].get("event") == "http_request"
+ )
+ assert completion["extra"]["request_id"] == response.headers["X-Request-ID"]
+ assert completion["extra"]["outcome"] == "success"
+ assert completion["extra"]["status_code"] == 200
+ assert completion["extra"]["duration_ms"] >= 0
+
+
@pytest.mark.skipif(not HAS_AIOHTTP, reason="aiohttp not installed")
@pytest.mark.asyncio
async def test_api_routes_allow_requests_without_configured_api_key(aiohttp_client, mock_agent) -> None:
diff --git a/tests/utils/test_log_config.py b/tests/utils/test_log_config.py
new file mode 100644
index 000000000..fd72294ba
--- /dev/null
+++ b/tests/utils/test_log_config.py
@@ -0,0 +1,40 @@
+from io import StringIO
+
+from loguru import logger
+
+from nanobot.utils.log_config import add_console_log_sink
+
+
+def test_console_log_sink_includes_context_and_escapes_message_newlines() -> None:
+ output = StringIO()
+ sink = add_console_log_sink(output, level="DEBUG")
+ try:
+ with logger.contextualize(
+ request_id="request-1",
+ turn_id="turn-1",
+ session_key="channel:chat",
+ ):
+ logger.bind(channel="test").info("first line\nsecond line")
+ finally:
+ logger.remove(sink)
+
+ rendered = output.getvalue()
+ assert "request=request-1 turn=turn-1 session=channel:chat" in rendered
+ assert "first line\\nsecond line" in rendered
+ assert "first line\nsecond line" not in rendered
+
+
+def test_console_log_sink_keeps_exception_traceback_multiline() -> None:
+ output = StringIO()
+ sink = add_console_log_sink(output)
+ try:
+ try:
+ raise RuntimeError("broken")
+ except RuntimeError:
+ logger.opt(exception=True).error("operation failed")
+ finally:
+ logger.remove(sink)
+
+ rendered = output.getvalue()
+ assert "operation failed" in rendered
+ assert "RuntimeError: broken" in rendered
diff --git a/tests/utils/test_rotating_output.py b/tests/utils/test_rotating_output.py
new file mode 100644
index 000000000..edbab7812
--- /dev/null
+++ b/tests/utils/test_rotating_output.py
@@ -0,0 +1,94 @@
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+from nanobot.utils.rotating_output import (
+ BACKGROUND_LOG_BACKUP_COUNT_ENV,
+ BACKGROUND_LOG_MAX_BYTES_ENV,
+ BACKGROUND_LOG_PATH_ENV,
+ RotatingTextOutput,
+)
+
+
+def test_rotating_output_enforces_size_and_backup_count(tmp_path: Path) -> None:
+ log_path = tmp_path / "gateway.log"
+ output = RotatingTextOutput(log_path, max_bytes=8, backup_count=2)
+
+ output.write("12345678")
+ output.write("new")
+ output.write("abcdefgh")
+ output.close()
+
+ assert log_path.read_text(encoding="utf-8") == "abcdefgh"
+ assert (tmp_path / "gateway.log.1").read_text(encoding="utf-8") == "new"
+ assert (tmp_path / "gateway.log.2").read_text(encoding="utf-8") == "12345678"
+ assert not (tmp_path / "gateway.log.3").exists()
+
+
+def test_rotating_output_rotates_oversized_existing_file_before_append(tmp_path: Path) -> None:
+ log_path = tmp_path / "gateway.log"
+ log_path.write_text("existing output", encoding="utf-8")
+ output = RotatingTextOutput(log_path, max_bytes=8, backup_count=1)
+
+ output.write("fresh")
+ output.close()
+
+ assert log_path.read_text(encoding="utf-8") == "fresh"
+ assert (tmp_path / "gateway.log.1").read_text(encoding="utf-8") == "existing output"
+
+
+def test_rotating_output_falls_back_when_active_log_cannot_be_renamed(
+ tmp_path: Path,
+ monkeypatch,
+) -> None:
+ log_path = tmp_path / "gateway.log"
+ log_path.write_text("existing", encoding="utf-8")
+ original_replace = Path.replace
+
+ def fail_active_log_rename(path: Path, target: Path) -> Path:
+ if path == log_path:
+ raise PermissionError("file is being followed")
+ return original_replace(path, target)
+
+ monkeypatch.setattr(Path, "replace", fail_active_log_rename)
+ output = RotatingTextOutput(log_path, max_bytes=8, backup_count=1)
+
+ output.write("fresh")
+ output.close()
+
+ assert log_path.read_text(encoding="utf-8") == "fresh"
+ assert (tmp_path / "gateway.log.1").read_text(encoding="utf-8") == "existing"
+
+
+def test_background_output_environment_redirects_stdout_and_stderr(tmp_path: Path) -> None:
+ log_path = tmp_path / "gateway.log"
+ env = os.environ.copy()
+ env[BACKGROUND_LOG_PATH_ENV] = str(log_path)
+ env[BACKGROUND_LOG_MAX_BYTES_ENV] = "1024"
+ env[BACKGROUND_LOG_BACKUP_COUNT_ENV] = "1"
+
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ (
+ "from nanobot.utils.rotating_output import "
+ "configure_background_output_from_env; "
+ "configure_background_output_from_env(); "
+ "print('stdout line'); "
+ "print('stderr line', file=__import__('sys').stderr)"
+ ),
+ ],
+ check=True,
+ capture_output=True,
+ env=env,
+ text=True,
+ )
+
+ assert result.stdout == ""
+ assert result.stderr == ""
+ assert log_path.read_text(encoding="utf-8").splitlines() == [
+ "stdout line",
+ "stderr line",
+ ]