Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(documents): preserve literal Excel missing-marker text

Rudy Celekli71db80b76634
diff --git a/agent/src/tools/doc_reader_tool.py b/agent/src/tools/doc_reader_tool.py
index 2354b09c0..0486b3e43 100644
--- a/agent/src/tools/doc_reader_tool.py
+++ b/agent/src/tools/doc_reader_tool.py
@@ -22,6 +22,7 @@ from typing import Any, Callable
from src.agent.progress import emit_progress
from src.agent.tools import BaseTool
from src.security.scanner import with_security_warnings
+from src.tools.ocr import get_ocr_engine, get_ocr_install_hint
from src.tools.path_utils import safe_document_path
_MAX_CHARS = 15000
@@ -44,7 +45,6 @@ _TEXT_EXTS = {
".dockerfile", ".makefile", ".cmake",
}
-from src.tools.ocr import get_ocr_engine, get_ocr_install_hint
_cached_ocr_engine = None
_cached_ocr_checked = False
@@ -271,7 +271,7 @@ def _read_excel(path: Path) -> str:
total=total_sheets,
message=f"sheet {name}",
)
- df = xls.parse(name, dtype=str)
+ df = xls.parse(name, dtype=str, keep_default_na=False, na_values=[""])
preview = df.head(100).to_string(index=False)
parts.append(f"--- Sheet: {name} ({len(df)} rows × {len(df.columns)} cols) ---\n{preview}")
sheet_info.append({"name": name, "rows": len(df), "cols": len(df.columns)})
diff --git a/agent/tests/test_doc_reader_excel_literal_text.py b/agent/tests/test_doc_reader_excel_literal_text.py
new file mode 100644
index 000000000..fc0573d73
--- /dev/null
+++ b/agent/tests/test_doc_reader_excel_literal_text.py
@@ -0,0 +1,66 @@
+"""Physical Excel text must not be reinterpreted as pandas missing-value markers."""
+
+import json
+import re
+
+import pytest
+
+from src.tools.doc_reader_tool import DocReaderTool
+
+openpyxl = pytest.importorskip("openpyxl")
+pytest.importorskip("pandas")
+
+
+def read_workbook(tmp_path, monkeypatch, rows, second_sheet=False):
+ monkeypatch.setenv("VIBE_TRADING_ALLOWED_FILE_ROOTS", str(tmp_path))
+ path = tmp_path / "report.xlsx"
+ workbook = openpyxl.Workbook()
+ sheet = workbook.active
+ sheet.title = "Regions"
+ sheet.append(["Value", "Label", "Count"])
+ for row in rows:
+ sheet.append(row)
+ if second_sheet:
+ workbook.create_sheet("Second").append(["Native second-sheet heading"])
+ workbook.save(path)
+ reopened = openpyxl.load_workbook(path)
+ assert list(reopened["Regions"].values)[1:] == [tuple(row) for row in rows]
+ reopened.close()
+ return json.loads(DocReaderTool().execute(file_path=str(path)))
+
+
[email protected]("literal", ["NA", "N/A", "NULL", "None", "null", "n/a"])
+def test_native_literal_strings_survive_the_document_reader(
+ tmp_path, monkeypatch, literal
+):
+ result = read_workbook(tmp_path, monkeypatch, [[literal, "Region marker", 42]])
+ assert result["status"] == "ok", result
+ assert re.search(
+ r"^\s*" + re.escape(literal) + r"\s+Region marker\s+42\s*$",
+ result["text"],
+ re.M,
+ )
+ assert result["sheets"] == [{"name": "Regions", "rows": 1, "cols": 3}]
+
+
+def test_real_blank_cells_keep_existing_nan_representation(tmp_path, monkeypatch):
+ result = read_workbook(tmp_path, monkeypatch, [[None, "Ordinary", 42]])
+ assert result["status"] == "ok"
+ assert re.search(r"^\s*NaN\s+Ordinary\s+42\s*$", result["text"], re.M)
+
+
+def test_ordinary_text_and_all_sheets_keep_their_contract(tmp_path, monkeypatch):
+ result = read_workbook(
+ tmp_path, monkeypatch, [["0012", "Ordinary", 42]], second_sheet=True
+ )
+ assert result["status"] == "ok"
+ assert "0012" in result["text"] and "Native second-sheet heading" in result["text"]
+ assert [s["name"] for s in result["sheets"]] == ["Regions", "Second"]
+ assert result["text"].index("Regions") < result["text"].index("Second")
+
+
+def test_existing_hundred_row_preview_is_unchanged(tmp_path, monkeypatch):
+ rows = [[f"ROW-{i:03d}", "Ordinary", i] for i in range(101)]
+ result = read_workbook(tmp_path, monkeypatch, rows)
+ assert result["sheets"][0]["rows"] == 101
+ assert "ROW-099" in result["text"] and "ROW-100" not in result["text"]