Skip to content

Your lightweight open-source personal agent.

fix(exec): preserve UTF-8 across streaming output chunks

Booty Destroyer 6969dfed706bc06e
diff --git a/nanobot/agent/tools/exec_session.py b/nanobot/agent/tools/exec_session.py
index 6598ff66b..3fc9ed541 100644
--- a/nanobot/agent/tools/exec_session.py
+++ b/nanobot/agent/tools/exec_session.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import codecs
import time
import uuid
from collections import deque
@@ -149,13 +150,14 @@ class _ExecSession:
) -> None:
if stream is None:
return
+ decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
while True:
chunk = await stream.read(4096)
- if not chunk:
- break
- text = chunk.decode("utf-8", errors="replace")
+ text = decoder.decode(chunk, final=not chunk)
async with self._lock:
buffer.append(text)
+ if not chunk:
+ break
async def write(self, chars: str) -> str | None:
if self.process.returncode is not None:
diff --git a/tests/tools/test_exec_session_tools.py b/tests/tools/test_exec_session_tools.py
index 69d215392..0b00168cf 100644
--- a/tests/tools/test_exec_session_tools.py
+++ b/tests/tools/test_exec_session_tools.py
@@ -20,6 +20,7 @@ from nanobot.agent.tools.exec_session import (
ExecSessionTool,
ListExecSessionsTool,
_BoundedOutputBuffer,
+ _ExecSession,
_SessionPoll,
_truncate_output,
)
@@ -151,6 +152,32 @@ def test_bounded_output_buffer_keeps_head_tail_and_exact_drop_count():
assert buffer.retained_chars == 0
+async def test_exec_session_preserves_utf8_across_stdout_and_stderr_chunks():
+ stdout = asyncio.StreamReader()
+ stderr = asyncio.StreamReader()
+ stdout_bytes = b"a" * 4095 + "你好🙂".encode() + b"\xe2\x82"
+ stderr_bytes = b"b" * 4094 + "🙂é".encode() + b"\xff"
+ stdout.feed_data(stdout_bytes[:4096])
+ stderr.feed_data(stderr_bytes[:4096])
+ session = _ExecSession(
+ session_id="utf8-output",
+ process=SimpleNamespace(stdout=stdout, stderr=stderr),
+ command="test",
+ cwd=".",
+ timeout=None,
+ )
+
+ await asyncio.sleep(0)
+ stdout.feed_data(stdout_bytes[4096:])
+ stderr.feed_data(stderr_bytes[4096:])
+ stdout.feed_eof()
+ stderr.feed_eof()
+ await asyncio.gather(session._stdout_task, session._stderr_task)
+
+ assert session._stdout.drain() == ("a" * 4095 + "你好🙂\ufffd", 0)
+ assert session._stderr.drain() == ("b" * 4094 + "🙂é\ufffd", 0)
+
+
def test_exec_session_bounds_unpolled_stdout_and_stderr(tmp_path):
async def run() -> tuple[int, int, str, int]:
manager = ExecSessionManager()