Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(tabular): pass a table through when one cell is past the csv field limit (#3618)

Lukas334317db5d38
diff --git a/headroom/transforms/tabular_ingest.py b/headroom/transforms/tabular_ingest.py
index 373b9c771..aabf4ad80 100644
--- a/headroom/transforms/tabular_ingest.py
+++ b/headroom/transforms/tabular_ingest.py
@@ -62,9 +62,21 @@ class TabularCompressionResult:
def parse_csv(content: str, delimiter: str = ",") -> tuple[list[str], list[list[str]]]:
- """Parse delimited text via the stdlib csv reader."""
+ """Parse delimited text via the stdlib csv reader.
+
+ Returns no rows when the reader refuses the text. The one that reaches real
+ content is ``csv.field_size_limit``, 128 KB per cell by default: a single
+ pasted document or base64 blob in one column raises
+ ``Error: field larger than field limit (131072)`` for the whole file.
+ Raising that limit is process-global state a library should not be setting
+ on its host, so the table is treated as non-tabular instead and the caller
+ passes the text through verbatim, as it already does for a ragged table.
+ """
reader = csv.reader(io.StringIO(content), delimiter=delimiter)
- parsed = [row for row in reader if any(cell.strip() for cell in row)]
+ try:
+ parsed = [row for row in reader if any(cell.strip() for cell in row)]
+ except csv.Error:
+ return [], []
if not parsed:
return [], []
headers = [h.strip() for h in parsed[0]]
diff --git a/tests/test_transforms_tabular.py b/tests/test_transforms_tabular.py
index 7b7d7f1d7..94341150d 100644
--- a/tests/test_transforms_tabular.py
+++ b/tests/test_transforms_tabular.py
@@ -201,6 +201,28 @@ def test_compress_passes_through_ragged_table(monkeypatch) -> None:
assert result.compressed == ragged
+def _csv_with_an_oversized_cell() -> str:
+ # csv.field_size_limit is 128 KB per cell; one pasted document, log excerpt
+ # or base64 blob in a column goes past it.
+ return "id,title,body\nl,short,ok\n2,long,{}\n".format("x" * 200_000)
+
+
+def test_parse_csv_gives_up_on_a_cell_past_the_field_size_limit() -> None:
+ headers, rows = parse_csv(_csv_with_an_oversized_cell())
+
+ # csv.Error: field larger than field limit (131072) before this.
+ assert (headers, rows) == ([], [])
+
+
+def test_compress_passes_through_a_table_with_an_oversized_cell() -> None:
+ content = _csv_with_an_oversized_cell()
+
+ result = TabularCompressor().compress(content)
+
+ assert not result.was_modified
+ assert result.compressed == content
+
+
def test_parse_tabular_returns_none_for_non_tabular() -> None:
assert parse_tabular("just a normal paragraph here") is None