Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tabular): pass tables with repeated header names through verbatim (#4278)
Raphael Fakhried3195beef1d
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex d4188725d..35dcfd0ef 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -157,6 +157,11 @@ def parse_tabular(width = len(headers)if any(len(row) != width for row in rows):return None+ # Repeated header names (a SQL join's `id,name,id,name`, or blank markdown+ # header cells) collide as record keys: each row keeps only the last+ # column of each name, and the others are dropped. Pass through verbatim.+ if len(set(headers)) != width:+ return None# The detector finds fixed-width columns by single-space gutters, but the# parser splits on 2+ spaces, so a table like GNU `ls -l` can come back as# one cell per line. That is not a table; leave it verbatim.diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex bbe0ea044..0bf2ad40f 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -313,6 +313,29 @@ def test_compress_passes_through_ragged_table(monkeypatch) -> None:assert result.compressed == ragged+_JOIN_CSV = "id,name,id,name\n1,alice,10,acme\n2,bob,20,globex\n3,carol,30,initech\n"+_BLANK_HEADER_MARKDOWN = (+ "| id | | |\n| --- | --- | --- |\n| 1 | alice | x |\n| 2 | bob | y |\n| 3 | carol | z |"+)++[email protected]("content", [_JOIN_CSV, _BLANK_HEADER_MARKDOWN])+def test_parse_tabular_rejects_duplicate_headers(content: str) -> None:+ # Records are keyed by header, so a repeated name keeps only its last+ # column: `alice` and `bob` would vanish from the compressed table.+ assert parse_tabular(content) is None+++def test_compress_passes_through_duplicate_headers() -> None:+ # Large enough that SmartCrusher would otherwise save bytes on it.+ content = "id,name,id,name\n" + "".join(+ f"{i},user_{i},{100 + i},org_{i % 3}\n" for i in range(40)+ )+ result = TabularCompressor().compress(content)+ assert not result.was_modified+ assert result.compressed == content++def _csv_with_an_oversized_cell() -> str:# csv.field_size_limit is 128 KB per cell; one pasted document, log excerpt# or base64 blob in a column goes past it.