Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(tool): keep Edit from rewriting non-UTF-8 files lossily (#3322)
Xinmin Zeng74b45545ff9c
diff --git a/src/agentscope/tool/_builtin/_edit.py b/src/agentscope/tool/_builtin/_edit.pyindex e247c8eab..9b804a051 100644--- a/src/agentscope/tool/_builtin/_edit.py+++ b/src/agentscope/tool/_builtin/_edit.py@@ -344,9 +344,7 @@ Usage:raw = await self._backend.read_file(file_path)# Normalize CRLF/CR to match the cached-content path and# the LF-based old_string the caller supplies.- content = _normalize_newlines(- raw.decode("utf-8", errors="replace"),- )+ content = _normalize_newlines(raw.decode("utf-8"))except Exception as e:return ToolChunk(content=[TextBlock(text=f"Error reading file: {str(e)}")],diff --git a/src/agentscope/tool/_builtin/_read.py b/src/agentscope/tool/_builtin/_read.pyindex 8903de865..936249606 100644--- a/src/agentscope/tool/_builtin/_read.py+++ b/src/agentscope/tool/_builtin/_read.py@@ -544,17 +544,24 @@ Usage:if cache is not None:lines = cache.lines+ lossy = Falseif lines is None:raw = await self._backend.read_file(file_path)- content_str = raw.decode("utf-8", errors="replace")+ try:+ content_str = raw.decode("utf-8")+ except UnicodeDecodeError:+ content_str = raw.decode("utf-8", errors="replace")+ lossy = True# Normalize CRLF/CR so cached lines end in "\n" regardless# of the platform the file was written on (Windows text# files use "\r\n").content_str = _normalize_newlines(content_str)lines = content_str.splitlines(keepends=True)- # Cache file if state is provided- if _agent_state is not None:+ # Edit rebuilds the whole file from the cache and Write takes+ # it as proof the file was seen, so a lossy decode must not+ # populate it.+ if _agent_state is not None and not lossy:await _agent_state.tool_context.cache_file(file_path=file_path,lines=lines,@@ -585,6 +592,12 @@ Usage:# Join all linesresult = "\n".join(formatted_lines)+ if lossy:+ result += (+ "\n\nNote: This file is not valid UTF-8. Invalid bytes "+ "are shown as U+FFFD, and the file cannot be modified "+ "with the Edit tool."+ )return ToolChunk(content=[TextBlock(text=result)],diff --git a/tests/builtin_edit_test.py b/tests/builtin_edit_test.pyindex 3e8f31650..58d88a855 100644--- a/tests/builtin_edit_test.py+++ b/tests/builtin_edit_test.py@@ -240,6 +240,45 @@ class EditToolTest(IsolatedAsyncioTestCase):)self.assertGreater(len(suggestions), 0)+ async def test_non_utf8_file_is_not_edited(self) -> None:+ """Test editing a non-UTF-8 file fails and keeps its bytes."""+ original = b"name = 'caf\xe9'\ncount = 1\n"+ with open(self.temp_file.name, "wb") as f:+ f.write(original)++ chunk = await self.edit_tool(+ file_path=self.temp_file.name,+ old_string="count = 1",+ new_string="count = 2",+ )++ self.assertEqual(+ chunk.model_dump(+ exclude={+ "id": True,+ "content": {"__all__": {"id", "created_at"}},+ },+ ),+ {+ "content": [+ {+ "type": "text",+ "text": (+ "Error reading file: 'utf-8' codec can't decode "+ "byte 0xe9 in position 11: invalid continuation "+ "byte"+ ),+ "finished_at": None,+ },+ ],+ "state": "error",+ "is_last": True,+ "metadata": {},+ },+ )+ with open(self.temp_file.name, "rb") as f:+ self.assertEqual(f.read(), original)+async def test_permission_suggestions_escape_parent(self) -> None:"""Test approved suggestions cover only the literal parent."""for parent, escaped, sibling in (diff --git a/tests/builtin_file_cache_test.py b/tests/builtin_file_cache_test.pyindex 523ed26bf..3aa20796b 100644--- a/tests/builtin_file_cache_test.py+++ b/tests/builtin_file_cache_test.py@@ -607,3 +607,107 @@ class FileCacheTest( # pylint: disable=too-many-public-methodswith open(self.test_file, "r", encoding="utf-8") as f:content = f.read()self.assertEqual(content, "final\n")++ async def test_non_utf8_read_cannot_back_edit_or_write(self) -> None:+ """Test a lossy non-UTF-8 Read cannot back an Edit or Write."""+ original = b"name = 'caf\xe9'\ncount = 1\n"+ with open(self.test_file, "wb") as f:+ f.write(original)++ read_chunk = await self.read_tool(+ file_path=self.test_file,+ _agent_state=self.state,+ )+ self.assertEqual(+ read_chunk.model_dump(+ exclude={+ "id": True,+ "content": {"__all__": {"id", "created_at"}},+ },+ ),+ {+ "content": [+ {+ "type": "text",+ "text": (+ " 1\tname = 'caf\ufffd'\n"+ " 2\tcount = 1\n\n"+ "Note: This file is not valid UTF-8. Invalid "+ "bytes are shown as U+FFFD, and the file cannot "+ "be modified with the Edit tool."+ ),+ "finished_at": None,+ },+ ],+ "state": "running",+ "is_last": True,+ "metadata": {},+ },+ )+ self.assertEqual(self.state.tool_context.read_file_cache, [])++ edit_chunk = await self.edit_tool(+ file_path=self.test_file,+ old_string="count = 1",+ new_string="count = 2",+ _agent_state=self.state,+ )+ self.assertEqual(+ edit_chunk.model_dump(+ exclude={+ "id": True,+ "content": {"__all__": {"id", "created_at"}},+ },+ ),+ {+ "content": [+ {+ "type": "text",+ "text": (+ "Error: To edit a file, you must first read "+ "it using the Read tool."+ ),+ "finished_at": None,+ },+ ],+ "state": "error",+ "is_last": True,+ "metadata": {},+ },+ )++ with open(self.test_file, "rb") as f:+ self.assertEqual(f.read(), original)++ write_chunk = await self.write_tool(+ file_path=self.test_file,+ content="replacement\n",+ _agent_state=self.state,+ )+ self.assertEqual(+ write_chunk.model_dump(+ exclude={+ "id": True,+ "content": {"__all__": {"id", "created_at"}},+ },+ ),+ {+ "content": [+ {+ "type": "text",+ "text": (+ f"Error: File {self.test_file} exists but has "+ "not been read yet. You must read the file "+ "first before writing to it."+ ),+ "finished_at": None,+ },+ ],+ "state": "error",+ "is_last": True,+ "metadata": {},+ },+ )++ with open(self.test_file, "rb") as f:+ self.assertEqual(f.read(), original)