Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(tabular): pass a table through when one cell is past the csv field limit (#3618)
Lukas334317db5d38
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.pyindex 373b9c771..aabf4ad80 100644--- a/headroom/transforms/tabular_ingest.py+++ b/headroom/transforms/tabular_ingest.py@@ -62,9 +62,21 @@ class TabularCompressionResult:def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[str]]]:- """Parse delimited text via the stdlib csv reader."""+ """Parse delimited text via the stdlib csv reader.++ Returns no rows when the reader refuses the text. The one that reaches real+ content is ``csv.field_size_limit``, 128 KB per cell by default: a single+ pasted document or base64 blob in one column raises+ ``Error: field larger than field limit (131072)`` for the whole file.+ Raising that limit is process-global state a library should not be setting+ on its host, so the table is treated as non-tabular instead and the caller+ passes the text through verbatim, as it already does for a ragged table.+ """reader = csv.reader(io.StringIO(content), delimiter=delimiter)- parsed = [row for row in reader if any(cell.strip() for cell in row)]+ try:+ parsed = [row for row in reader if any(cell.strip() for cell in row)]+ except csv.Error:+ return [], []if not parsed:return [], []headers = [h.strip() for h in parsed[0]]diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.pyindex 7b7d7f1d7..94341150d 100644--- a/tests/test_transforms_tabular.py+++ b/tests/test_transforms_tabular.py@@ -201,6 +201,28 @@ def test_compress_passes_through_ragged_table(monkeypatch) -> None:assert result.compressed == ragged+def _csv_with_an_oversized_cell() -> str:+ # csv.field_size_limit is 128 KB per cell; one pasted document, log excerpt+ # or base64 blob in a column goes past it.+ return "id,title,body\nl,short,ok\n2,long,{}\n".format("x" * 200_000)+++def test_parse_csv_gives_up_on_a_cell_past_the_field_size_limit() -> None:+ headers, rows = parse_csv(_csv_with_an_oversized_cell())++ # csv.Error: field larger than field limit (131072) before this.+ assert (headers, rows) == ([], [])+++def test_compress_passes_through_a_table_with_an_oversized_cell() -> None:+ content = _csv_with_an_oversized_cell()++ result = TabularCompressor().compress(content)++ assert not result.was_modified+ assert result.compressed == content++def test_parse_tabular_returns_none_for_non_tabular() -> None:assert parse_tabular("just a normal paragraph here") is None