"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(tools): decode UTF-16 BOM text before latin-1 fallback
santhreal8a4f6260bb9c
diff --git a/agent/src/tools/doc_reader_tool.py b/agent/src/tools/doc_reader_tool.pyindex c9e733122..2354b09c0 100644--- a/agent/src/tools/doc_reader_tool.py+++ b/agent/src/tools/doc_reader_tool.py@@ -353,6 +353,14 @@ def _read_image(path: Path) -> str:def _read_text(path: Path) -> str:"""Read a text-like file with encoding fallback."""data = path.read_bytes()+ # UTF-16 with BOM (Notepad "Unicode", Excel Unicode Text) before latin-1:+ # latin-1 never raises, so BOM+NUL bytes become mojibake without this.+ if data.startswith((b"\xff\xfe", b"\xfe\xff")):+ try:+ decoded = data.decode("utf-16")+ return _envelope(path, "text", decoded, encoding="utf-16", size=len(data))+ except UnicodeDecodeError:+ passlast_err: Exception | None = Nonefor enc in _ENCODING_FALLBACK:try:diff --git a/agent/tests/test_doc_reader.py b/agent/tests/test_doc_reader.pyindex 8794ab358..8be2196cd 100644--- a/agent/tests/test_doc_reader.py+++ b/agent/tests/test_doc_reader.py@@ -63,6 +63,19 @@ def test_gbk_fallback(tmp_path: Path) -> None:assert "中文" in result["text"]+def test_utf16_bom_not_latin1_mojibake(tmp_path: Path) -> None:+ # Notepad/Excel "Unicode text" writes UTF-16 with BOM. latin-1 never+ # raises, so without utf-16 earlier in the fallback the file was accepted+ # as mojibake (NUL bytes between ASCII letters).+ p = tmp_path / "unicode.txt"+ p.write_bytes("hello 动量\n".encode("utf-16"))+ result = _call(p)+ assert result["status"] == "ok"+ assert result["encoding"] == "utf-16"+ assert result["text"] == "hello 动量\n"+ assert "\x00" not in result["text"]++def test_unknown_extension_treated_as_text(tmp_path: Path) -> None:p = tmp_path / "weird.xyzfmt"p.write_text("abc", encoding="utf-8")