Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms): drop trailing empty spreadsheet rows and stray CR in CSV render (#3254)
Abhay Singh10d8f15bc7a3
diff --git a/headroom/transforms/spreadsheet_ingest.py b/headroom/transforms/spreadsheet_ingest.pyindex 01bad93e2..14d6c2e5c 100644--- a/headroom/transforms/spreadsheet_ingest.py+++ b/headroom/transforms/spreadsheet_ingest.py@@ -18,11 +18,30 @@ from pathlib import Path__all__ = ["load_spreadsheet"]+def _is_blank_row(row: list[object]) -> bool:+ return all(cell is None or str(cell).strip() == "" for cell in row)++def _rows_to_csv(rows: list[list[object]]) -> str:- """Render rows to CSV text, dropping fully empty trailing rows."""+ """Render rows to CSV text, dropping fully empty trailing rows.++ openpyxl's ``iter_rows`` walks the sheet's *used range*, which routinely+ extends past the last data row (leftover formatting, a cleared cell), so a+ real sheet commonly ends in ``(None, None, ...)`` tuples. Those were written+ out as blank ``,`` rows — contradicting this function's own contract — so the+ trailing empties are dropped here first.++ ``lineterminator="\\n"`` is load-bearing: ``csv.writer`` defaults to+ ``\\r\\n``, and the old ``.strip("\\n")`` removed only the ``\\n``, leaving a+ dangling ``\\r`` on the final line (and ``,\\r`` residue behind each undropped+ empty row). Using ``\\n`` makes the strip clean.+ """+ last = len(rows)+ while last > 0 and _is_blank_row(rows[last - 1]):+ last -= 1buf = io.StringIO()- writer = csv.writer(buf)- for row in rows:+ writer = csv.writer(buf, lineterminator="\n")+ for row in rows[:last]:writer.writerow(["" if cell is None else cell for cell in row])return buf.getvalue().strip("\n")diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex 9316ad22f..7b7d7f1d7 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -322,6 +322,29 @@ def test_router_respects_disable_flag() -> None:# Binary spreadsheet ingestion -----------------------------------------------+def test_rows_to_csv_drops_trailing_empty_rows_and_has_no_dangling_cr() -> None:+ """Trailing all-empty rows are dropped and the output has no stray ``\\r``.++ openpyxl's used-range routinely extends past the last data row, so a sheet+ commonly ends in ``(None, None, ...)`` tuples. Those were emitted as blank+ ``,`` rows, and ``csv.writer``'s default ``\\r\\n`` terminator combined with+ ``.strip("\\n")`` left a dangling ``\\r`` — noise fed straight to the LLM.+ """+ from headroom.transforms.spreadsheet_ingest import _rows_to_csv++ rendered = _rows_to_csv(+ [["Name", "Age"], ["Alice", "30"], [None, None], ["", " "], [None, None]]+ )+ assert rendered == "Name,Age\nAlice,30"+ assert "\r" not in rendered++ # Interior empty rows are preserved (only the trailing run is dropped).+ assert _rows_to_csv([["a", "b"], [None, None], ["c", "d"], [None, None]]) == "a,b\n,\nc,d"++ # A fully empty sheet renders to the empty string.+ assert _rows_to_csv([[None, None], ["", ""]]) == ""++@pytest.mark.skipif(not _HAS_OPENPYXL, reason="openpyxl not installed")def test_load_and_compress_xlsx(tmp_path) -> None:import openpyxl