Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tabular): pass tables with repeated header names through verbatim (#4278)

Raphael Fakhried3195beef1d
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index d4188725d..35dcfd0ef 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -157,6 +157,11 @@ def parse_tabular(
width = len(headers)
if any(len(row) != width for row in rows):
return None
+ # Repeated header names (a SQL join's `id,name,id,name`, or blank markdown
+ # header cells) collide as record keys: each row keeps only the last
+ # column of each name, and the others are dropped. Pass through verbatim.
+ if len(set(headers)) != width:
+ return None
# The detector finds fixed-width columns by single-space gutters, but the
# parser splits on 2+ spaces, so a table like GNU `ls -l` can come back as
# one cell per line. That is not a table; leave it verbatim.
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index bbe0ea044..0bf2ad40f 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -313,6 +313,29 @@ def test_compress_passes_through_ragged_table(monkeypatch) -> None:
assert result.compressed == ragged
+_JOIN_CSV = "id,name,id,name\n1,alice,10,acme\n2,bob,20,globex\n3,carol,30,initech\n"
+_BLANK_HEADER_MARKDOWN = (
+ "| id | | |\n| --- | --- | --- |\n| 1 | alice | x |\n| 2 | bob | y |\n| 3 | carol | z |"
+)
+
+
[email protected]("content", [_JOIN_CSV, _BLANK_HEADER_MARKDOWN])
+def test_parse_tabular_rejects_duplicate_headers(content: str) -> None:
+ # Records are keyed by header, so a repeated name keeps only its last
+ # column: `alice` and `bob` would vanish from the compressed table.
+ assert parse_tabular(content) is None
+
+
+def test_compress_passes_through_duplicate_headers() -> None:
+ # Large enough that SmartCrusher would otherwise save bytes on it.
+ content = "id,name,id,name\n" + "".join(
+ f"{i},user_{i},{100 + i},org_{i % 3}\n" for i in range(40)
+ )
+ result = TabularCompressor().compress(content)
+ assert not result.was_modified
+ assert result.compressed == content
+
+
def _csv_with_an_oversized_cell() -> str:
# csv.field_size_limit is 128 KB per cell; one pasted document, log excerpt
# or base64 blob in a column goes past it.