Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(tools): decode UTF-16 BOM text before latin-1 fallback

santhreal8a4f6260bb9c
diff --git a/agent/src/tools/doc_reader_tool.py b/agent/src/tools/doc_reader_tool.py
index c9e733122..2354b09c0 100644
--- a/agent/src/tools/doc_reader_tool.py
+++ b/agent/src/tools/doc_reader_tool.py
@@ -353,6 +353,14 @@ def _read_image(path: Path) -> str:
def _read_text(path: Path) -> str:
"""Read a text-like file with encoding fallback."""
data = path.read_bytes()
+ # UTF-16 with BOM (Notepad "Unicode", Excel Unicode Text) before latin-1:
+ # latin-1 never raises, so BOM+NUL bytes become mojibake without this.
+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+ try:
+ decoded = data.decode("utf-16")
+ return _envelope(path, "text", decoded, encoding="utf-16", size=len(data))
+ except UnicodeDecodeError:
+ pass
last_err: Exception | None = None
for enc in _ENCODING_FALLBACK:
try:
diff --git a/agent/tests/test_doc_reader.py b/agent/tests/test_doc_reader.py
index 8794ab358..8be2196cd 100644
--- a/agent/tests/test_doc_reader.py
+++ b/agent/tests/test_doc_reader.py
@@ -63,6 +63,19 @@ def test_gbk_fallback(tmp_path: Path) -> None:
assert "中文" in result["text"]
+def test_utf16_bom_not_latin1_mojibake(tmp_path: Path) -> None:
+ # Notepad/Excel "Unicode text" writes UTF-16 with BOM. latin-1 never
+ # raises, so without utf-16 earlier in the fallback the file was accepted
+ # as mojibake (NUL bytes between ASCII letters).
+ p = tmp_path / "unicode.txt"
+ p.write_bytes("hello 动量\n".encode("utf-16"))
+ result = _call(p)
+ assert result["status"] == "ok"
+ assert result["encoding"] == "utf-16"
+ assert result["text"] == "hello 动量\n"
+ assert "\x00" not in result["text"]
+
+
def test_unknown_extension_treated_as_text(tmp_path: Path) -> None:
p = tmp_path / "weird.xyzfmt"
p.write_text("abc", encoding="utf-8")