Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tabular): keep interior blank CSV rows instead of dropping them (#4563)
HetCreep8c360ca69fd3
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex 32967136c..2ea713601 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -77,8 +77,18 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[parsed = [row for row in reader if row and not (len(row) == 1 and not row[0].strip())]except csv.Error:return [], []- if not parsed:+ # Trim all-blank rows at both ends, but keep interior ones so the row+ # count survives, like spreadsheet_ingest._rows_to_csv. One pass and one+ # slice: popping index 0 per leading blank row is quadratic. An interior+ # all-blank row is normalized to empty cells, so a whitespace-only row and+ # an empty row compress the same.+ filled = [i for i, row in enumerate(parsed) if any(cell.strip() for cell in row)]+ if not filled:return [], []+ parsed = [+ row if any(cell.strip() for cell in row) else [""] * len(row)+ for row in parsed[filled[0] : filled[-1] + 1]+ ]headers = [h.strip() for h in parsed[0]]return headers, parsed[1:]diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex 9b80b17c0..fd8b28ce2 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -403,6 +403,37 @@ def test_parse_csv_and_records() -> None:assert records[1] == {"name": "Bob", "age": "25", "city": "LA"}+def test_parse_csv_keeps_interior_blank_rows_and_trims_the_ends() -> None:+ headers, rows = parse_csv("a,b\n1,2\n,\n3,4\n,\n5,6\n,\n")+ assert headers == ["a", "b"]+ assert rows == [["1", "2"], ["", ""], ["3", "4"], ["", ""], ["5", "6"]]+ # Empty and whitespace-only lines have no real cells and are still skipped.+ assert parse_csv("a,b\n1,2\n\n3,4\n")[1] == [["1", "2"], ["3", "4"]]+ assert parse_csv("a,b\n1,2\n \n3,4\n")[1] == [["1", "2"], ["3", "4"]]+++def test_parse_csv_normalizes_whitespace_only_interior_rows() -> None:+ headers, rows = parse_csv("a,b\n1,2\n , \n3,4\n")+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]+++def test_parse_csv_trims_a_large_blank_prefix_in_linear_time() -> None:+ import time++ def run(n: int) -> float:+ text = ",\n" * n + "a,b\n1,2\n"+ t0 = time.perf_counter()+ headers, rows = parse_csv(text)+ elapsed = time.perf_counter() - t0+ assert headers == ["a", "b"]+ assert rows == [["1", "2"]]+ return elapsed++ small, large = run(25_000), run(200_000)+ # 8x the rows: linear stays near 8x; the old pop(0) loop was ~57x.+ assert large < max(small, 1e-3) * 25, (small, large)++def test_parse_markdown_table_drops_separator() -> None:headers, rows = parse_markdown_table(MARKDOWN)assert headers == ["name", "age"]