Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(transforms): pass through ragged tables instead of misaligning columns (#1713)
Parideboyc7665ca08863
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex 07b889b13..373b9c771 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -134,6 +134,13 @@ def parse_tabular(if not headers or not rows:return None+ # Ragged tables (rows whose cell count differs from the header count)+ # can't be zipped into records without shifting values under the wrong+ # column — a compressed table must never state facts the original+ # didn't (#1652). Treat them as non-tabular and pass through verbatim.+ width = len(headers)+ if any(len(row) != width for row in rows):+ return Nonereturn headers, rows, fmtdiff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex 858708892..9316ad22f 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -153,6 +153,54 @@ def test_parse_markdown_table_drops_separator() -> None:assert all("---" not in cell for row in rows for cell in row)+def test_parse_tabular_rejects_ragged_fixed_width(monkeypatch) -> None:+ # Rows with differing cell counts can't be zipped under the headers+ # without misattributing columns (#1652) — must pass through.+ import headroom.transforms.tabular_ingest as ti++ monkeypatch.setattr(+ ti,+ "detect_content_type",+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}),+ )+ ragged = (+ "tool installed latest status\n"+ "rtk 0.42.4 0.43.0 update available\n"+ "rtk ✓ 0.42.4 0.42.4 - up-to-date"+ )+ assert ti.parse_tabular(ragged) is None+++def test_parse_tabular_rejects_ragged_markdown(monkeypatch) -> None:+ import headroom.transforms.tabular_ingest as ti++ monkeypatch.setattr(+ ti,+ "detect_content_type",+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "markdown"}),+ )+ ragged = "| a | b | c |\n| --- | --- | --- |\n| 1 | 2 | 3 |\n| 4 | 5 |"+ assert ti.parse_tabular(ragged) is None+++def test_compress_passes_through_ragged_table(monkeypatch) -> None:+ import headroom.transforms.tabular_ingest as ti++ monkeypatch.setattr(+ ti,+ "detect_content_type",+ lambda _c: DetectionResult(ContentType.TABULAR, 0.9, {"format": "fixed_width"}),+ )+ ragged = (+ "tool installed latest status\n"+ "rtk 0.42.4 0.43.0 update available\n"+ "rtk ✓ 0.42.4 0.42.4 - up-to-date"+ )+ result = TabularCompressor().compress(ragged)+ assert not result.was_modified+ assert result.compressed == ragged++def test_parse_tabular_returns_none_for_non_tabular() -> None:assert parse_tabular("just a normal paragraph here") is None