Skip to content

Build and run agents you can see, understand and trust.

fix(tool): keep Edit from rewriting non-UTF-8 files lossily (#3322)

Xinmin Zeng74b45545ff9c
diff --git a/src/agentscope/tool/_builtin/_edit.py b/src/agentscope/tool/_builtin/_edit.py
index e247c8eab..9b804a051 100644
--- a/src/agentscope/tool/_builtin/_edit.py
+++ b/src/agentscope/tool/_builtin/_edit.py
@@ -344,9 +344,7 @@ Usage:
raw = await self._backend.read_file(file_path)
# Normalize CRLF/CR to match the cached-content path and
# the LF-based old_string the caller supplies.
- content = _normalize_newlines(
- raw.decode("utf-8", errors="replace"),
- )
+ content = _normalize_newlines(raw.decode("utf-8"))
except Exception as e:
return ToolChunk(
content=[TextBlock(text=f"Error reading file: {str(e)}")],
diff --git a/src/agentscope/tool/_builtin/_read.py b/src/agentscope/tool/_builtin/_read.py
index 8903de865..936249606 100644
--- a/src/agentscope/tool/_builtin/_read.py
+++ b/src/agentscope/tool/_builtin/_read.py
@@ -544,17 +544,24 @@ Usage:
if cache is not None:
lines = cache.lines
+ lossy = False
if lines is None:
raw = await self._backend.read_file(file_path)
- content_str = raw.decode("utf-8", errors="replace")
+ try:
+ content_str = raw.decode("utf-8")
+ except UnicodeDecodeError:
+ content_str = raw.decode("utf-8", errors="replace")
+ lossy = True
# Normalize CRLF/CR so cached lines end in "\n" regardless
# of the platform the file was written on (Windows text
# files use "\r\n").
content_str = _normalize_newlines(content_str)
lines = content_str.splitlines(keepends=True)
- # Cache file if state is provided
- if _agent_state is not None:
+ # Edit rebuilds the whole file from the cache and Write takes
+ # it as proof the file was seen, so a lossy decode must not
+ # populate it.
+ if _agent_state is not None and not lossy:
await _agent_state.tool_context.cache_file(
file_path=file_path,
lines=lines,
@@ -585,6 +592,12 @@ Usage:
# Join all lines
result = "\n".join(formatted_lines)
+ if lossy:
+ result += (
+ "\n\nNote: This file is not valid UTF-8. Invalid bytes "
+ "are shown as U+FFFD, and the file cannot be modified "
+ "with the Edit tool."
+ )
return ToolChunk(
content=[TextBlock(text=result)],
diff --git a/tests/builtin_edit_test.py b/tests/builtin_edit_test.py
index 3e8f31650..58d88a855 100644
--- a/tests/builtin_edit_test.py
+++ b/tests/builtin_edit_test.py
@@ -240,6 +240,45 @@ class EditToolTest(IsolatedAsyncioTestCase):
)
self.assertGreater(len(suggestions), 0)
+ async def test_non_utf8_file_is_not_edited(self) -> None:
+ """Test editing a non-UTF-8 file fails and keeps its bytes."""
+ original = b"name = 'caf\xe9'\ncount = 1\n"
+ with open(self.temp_file.name, "wb") as f:
+ f.write(original)
+
+ chunk = await self.edit_tool(
+ file_path=self.temp_file.name,
+ old_string="count = 1",
+ new_string="count = 2",
+ )
+
+ self.assertEqual(
+ chunk.model_dump(
+ exclude={
+ "id": True,
+ "content": {"__all__": {"id", "created_at"}},
+ },
+ ),
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ "Error reading file: 'utf-8' codec can't decode "
+ "byte 0xe9 in position 11: invalid continuation "
+ "byte"
+ ),
+ "finished_at": None,
+ },
+ ],
+ "state": "error",
+ "is_last": True,
+ "metadata": {},
+ },
+ )
+ with open(self.temp_file.name, "rb") as f:
+ self.assertEqual(f.read(), original)
+
async def test_permission_suggestions_escape_parent(self) -> None:
"""Test approved suggestions cover only the literal parent."""
for parent, escaped, sibling in (
diff --git a/tests/builtin_file_cache_test.py b/tests/builtin_file_cache_test.py
index 523ed26bf..3aa20796b 100644
--- a/tests/builtin_file_cache_test.py
+++ b/tests/builtin_file_cache_test.py
@@ -607,3 +607,107 @@ class FileCacheTest( # pylint: disable=too-many-public-methods
with open(self.test_file, "r", encoding="utf-8") as f:
content = f.read()
self.assertEqual(content, "final\n")
+
+ async def test_non_utf8_read_cannot_back_edit_or_write(self) -> None:
+ """Test a lossy non-UTF-8 Read cannot back an Edit or Write."""
+ original = b"name = 'caf\xe9'\ncount = 1\n"
+ with open(self.test_file, "wb") as f:
+ f.write(original)
+
+ read_chunk = await self.read_tool(
+ file_path=self.test_file,
+ _agent_state=self.state,
+ )
+ self.assertEqual(
+ read_chunk.model_dump(
+ exclude={
+ "id": True,
+ "content": {"__all__": {"id", "created_at"}},
+ },
+ ),
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ " 1\tname = 'caf\ufffd'\n"
+ " 2\tcount = 1\n\n"
+ "Note: This file is not valid UTF-8. Invalid "
+ "bytes are shown as U+FFFD, and the file cannot "
+ "be modified with the Edit tool."
+ ),
+ "finished_at": None,
+ },
+ ],
+ "state": "running",
+ "is_last": True,
+ "metadata": {},
+ },
+ )
+ self.assertEqual(self.state.tool_context.read_file_cache, [])
+
+ edit_chunk = await self.edit_tool(
+ file_path=self.test_file,
+ old_string="count = 1",
+ new_string="count = 2",
+ _agent_state=self.state,
+ )
+ self.assertEqual(
+ edit_chunk.model_dump(
+ exclude={
+ "id": True,
+ "content": {"__all__": {"id", "created_at"}},
+ },
+ ),
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ "Error: To edit a file, you must first read "
+ "it using the Read tool."
+ ),
+ "finished_at": None,
+ },
+ ],
+ "state": "error",
+ "is_last": True,
+ "metadata": {},
+ },
+ )
+
+ with open(self.test_file, "rb") as f:
+ self.assertEqual(f.read(), original)
+
+ write_chunk = await self.write_tool(
+ file_path=self.test_file,
+ content="replacement\n",
+ _agent_state=self.state,
+ )
+ self.assertEqual(
+ write_chunk.model_dump(
+ exclude={
+ "id": True,
+ "content": {"__all__": {"id", "created_at"}},
+ },
+ ),
+ {
+ "content": [
+ {
+ "type": "text",
+ "text": (
+ f"Error: File {self.test_file} exists but has "
+ "not been read yet. You must read the file "
+ "first before writing to it."
+ ),
+ "finished_at": None,
+ },
+ ],
+ "state": "error",
+ "is_last": True,
+ "metadata": {},
+ },
+ )
+
+ with open(self.test_file, "rb") as f:
+ self.assertEqual(f.read(), original)