Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(transforms): drop trailing empty spreadsheet rows and stray CR in CSV render (#3254)

Abhay Singh10d8f15bc7a3
diff --git a/headroom/transforms/spreadsheet_ingest.py b/headroom/transforms/spreadsheet_ingest.py
index 01bad93e2..14d6c2e5c 100644
--- a/headroom/transforms/spreadsheet_ingest.py
+++ b/headroom/transforms/spreadsheet_ingest.py
@@ -18,11 +18,30 @@ from pathlib import Path
__all__ = ["load_spreadsheet"]
+def _is_blank_row(row: list[object]) -> bool:
+ return all(cell is None or str(cell).strip() == "" for cell in row)
+
+
def _rows_to_csv(rows: list[list[object]]) -> str:
- """Render rows to CSV text, dropping fully empty trailing rows."""
+ """Render rows to CSV text, dropping fully empty trailing rows.
+
+ openpyxl's ``iter_rows`` walks the sheet's *used range*, which routinely
+ extends past the last data row (leftover formatting, a cleared cell), so a
+ real sheet commonly ends in ``(None, None, ...)`` tuples. Those were written
+ out as blank ``,`` rows — contradicting this function's own contract — so the
+ trailing empties are dropped here first.
+
+ ``lineterminator="\\n"`` is load-bearing: ``csv.writer`` defaults to
+ ``\\r\\n``, and the old ``.strip("\\n")`` removed only the ``\\n``, leaving a
+ dangling ``\\r`` on the final line (and ``,\\r`` residue behind each undropped
+ empty row). Using ``\\n`` makes the strip clean.
+ """
+ last = len(rows)
+ while last > 0 and _is_blank_row(rows[last - 1]):
+ last -= 1
buf = io.StringIO()
- writer = csv.writer(buf)
- for row in rows:
+ writer = csv.writer(buf, lineterminator="\n")
+ for row in rows[:last]:
writer.writerow(["" if cell is None else cell for cell in row])
return buf.getvalue().strip("\n")
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index 9316ad22f..7b7d7f1d7 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -322,6 +322,29 @@ def test_router_respects_disable_flag() -> None:
# Binary spreadsheet ingestion -----------------------------------------------
+def test_rows_to_csv_drops_trailing_empty_rows_and_has_no_dangling_cr() -> None:
+ """Trailing all-empty rows are dropped and the output has no stray ``\\r``.
+
+ openpyxl's used-range routinely extends past the last data row, so a sheet
+ commonly ends in ``(None, None, ...)`` tuples. Those were emitted as blank
+ ``,`` rows, and ``csv.writer``'s default ``\\r\\n`` terminator combined with
+ ``.strip("\\n")`` left a dangling ``\\r`` — noise fed straight to the LLM.
+ """
+ from headroom.transforms.spreadsheet_ingest import _rows_to_csv
+
+ rendered = _rows_to_csv(
+ [["Name", "Age"], ["Alice", "30"], [None, None], ["", " "], [None, None]]
+ )
+ assert rendered == "Name,Age\nAlice,30"
+ assert "\r" not in rendered
+
+ # Interior empty rows are preserved (only the trailing run is dropped).
+ assert _rows_to_csv([["a", "b"], [None, None], ["c", "d"], [None, None]]) == "a,b\n,\nc,d"
+
+ # A fully empty sheet renders to the empty string.
+ assert _rows_to_csv([[None, None], ["", ""]]) == ""
+
+
@pytest.mark.skipif(not _HAS_OPENPYXL, reason="openpyxl not installed")
def test_load_and_compress_xlsx(tmp_path) -> None:
import openpyxl