Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tabular): keep empty records and separator-like Markdown body rows (#4582)

Parideboy5cebb90db1f1
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index 35dcfd0ef..32967136c 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -74,7 +74,7 @@ def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[
"""
reader = csv.reader(io.StringIO(content), delimiter=delimiter)
try:
- parsed = [row for row in reader if any(cell.strip() for cell in row)]
+ parsed = [row for row in reader if row and not (len(row) == 1 and not row[0].strip())]
except csv.Error:
return [], []
if not parsed:
@@ -97,7 +97,10 @@ def parse_markdown_table(content: str) -> tuple[list[str], list[list[str]]]:
if len(lines) < 2:
return [], []
headers = split_row(lines[0])
- rows = [split_row(ln) for ln in lines[1:] if not is_separator(ln)]
+ body_lines = lines[1:]
+ if body_lines and is_separator(body_lines[0]):
+ body_lines = body_lines[1:]
+ rows = [split_row(ln) for ln in body_lines]
return headers, rows
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index e465db8bc..d1f3b962b 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -410,6 +410,30 @@ def test_parse_markdown_table_drops_separator() -> None:
assert all("---" not in cell for row in rows for cell in row)
+def test_parse_csv_keeps_empty_records() -> None:
+ headers, rows = parse_csv("a,b\n1,2\n,\n3,4")
+ assert headers == ["a", "b"]
+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]
+
+
+def test_parse_csv_keeps_empty_records_tsv() -> None:
+ headers, rows = parse_csv("a\tb\n1\t2\n\t\n3\t4", delimiter="\t")
+ assert headers == ["a", "b"]
+ assert rows == [["1", "2"], ["", ""], ["3", "4"]]
+
+
+def test_parse_csv_keeps_quoted_empty_record() -> None:
+ headers, rows = parse_csv('a,b\n"",""\n1,2')
+ assert headers == ["a", "b"]
+ assert rows[0] == ["", ""]
+
+
+def test_parse_markdown_table_keeps_separator_like_body_row() -> None:
+ headers, rows = parse_markdown_table("| a | b |\n|---|---|\n| 1 | 2 |\n| --- | --- |")
+ assert headers == ["a", "b"]
+ assert rows == [["1", "2"], ["---", "---"]]
+
+
def test_parse_tabular_rejects_ragged_fixed_width(monkeypatch) -> None:
# Rows with differing cell counts can't be zipped under the headers
# without misattributing columns (#1652) — must pass through.
@@ -562,7 +586,7 @@ def test_parse_tabular_rejects_single_column_fixed_width(monkeypatch) -> None:
def test_parse_tabular_none_when_no_data_rows_survive() -> None:
# Detected as a markdown table, but it is header + separator rows only:
# nothing survives as a data row, so parse_tabular bails to None.
- assert parse_tabular("| a | b |\n| --- | --- |\n| --- | --- |") is None
+ assert parse_tabular("| a | b |\n| --- | --- |") is None
def test_compression_ratio_zero_for_empty_original() -> None: