Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tabular): keep interior blank CSV rows instead of dropping them (#4563)

HetCreep8c360ca69fd3
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index 32967136c..2ea713601 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -77,8 +77,18 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[
parsed = [row for row in reader if row and not (len(row) == 1 and not row[0].strip())]
except csv.Error:
return [], []
- if not parsed:
+ # Trim all-blank rows at both ends, but keep interior ones so the row
+ # count survives, like spreadsheet_ingest._rows_to_csv. One pass and one
+ # slice: popping index 0 per leading blank row is quadratic. An interior
+ # all-blank row is normalized to empty cells, so a whitespace-only row and
+ # an empty row compress the same.
+ filled = [i for i, row in enumerate(parsed) if any(cell.strip() for cell in row)]
+ if not filled:
return [], []
+ parsed = [
+ row if any(cell.strip() for cell in row) else [""] * len(row)
+ for row in parsed[filled[0] : filled[-1] + 1]
+ ]
headers = [h.strip() for h in parsed[0]]
return headers, parsed[1:]
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index 9b80b17c0..fd8b28ce2 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -403,6 +403,37 @@ def test_parse_csv_and_records() -> None:
assert records[1] == {"name": "Bob", "age": "25", "city": "LA"}
+def test_parse_csv_keeps_interior_blank_rows_and_trims_the_ends() -> None:
+ headers, rows = parse_csv("a,b\n1,2\n,\n3,4\n,\n5,6\n,\n")
+ assert headers == ["a", "b"]
+ assert rows == [["1", "2"], ["", ""], ["3", "4"], ["", ""], ["5", "6"]]
+ # Empty and whitespace-only lines have no real cells and are still skipped.
+ assert parse_csv("a,b\n1,2\n\n3,4\n")[1] == [["1", "2"], ["3", "4"]]
+ assert parse_csv("a,b\n1,2\n \n3,4\n")[1] == [["1", "2"], ["3", "4"]]
+
+
+def test_parse_csv_normalizes_whitespace_only_interior_rows() -> None:
+ headers, rows = parse_csv("a,b\n1,2\n , \n3,4\n")
+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]
+
+
+def test_parse_csv_trims_a_large_blank_prefix_in_linear_time() -> None:
+ import time
+
+ def run(n: int) -> float:
+ text = ",\n" * n + "a,b\n1,2\n"
+ t0 = time.perf_counter()
+ headers, rows = parse_csv(text)
+ elapsed = time.perf_counter() - t0
+ assert headers == ["a", "b"]
+ assert rows == [["1", "2"]]
+ return elapsed
+
+ small, large = run(25_000), run(200_000)
+ # 8x the rows: linear stays near 8x; the old pop(0) loop was ~57x.
+ assert large < max(small, 1e-3) * 25, (small, large)
+
+
def test_parse_markdown_table_drops_separator() -> None:
headers, rows = parse_markdown_table(MARKDOWN)
assert headers == ["name", "age"]