Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(codec): reject densified blocks with mismatched row counts (#4271)
Andrew Barnes75df01d627cf
diff --git a/headroom/transforms/compaction_codec.py b/headroom/transforms/compaction_codec.pyindex 5413cb7a8..41933630d 100644--- a/headroom/transforms/compaction_codec.py+++ b/headroom/transforms/compaction_codec.py@@ -362,8 +362,9 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:"""Reconstruct the original array of objects from densified ``text``.Returns the list of dicts, or ``None`` if ``text`` is not a parseable- densified block or carries an irreversible removal marker (CCR / dropped- rows) — callers treat ``None`` as "cannot losslessly reverse".+ densified block, its row count differs from the header, or it carries an+ irreversible removal marker (CCR / dropped rows). Callers treat ``None``+ as "cannot losslessly reverse"."""if not is_compacted(text) or contains_removal_marker(text):return None@@ -390,8 +391,12 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:return Nonelegends[name[1:]] = result+ rows = _split_data_rows(rest, declared)+ if len(rows) != declared:+ return None+out: list[dict[str, Any]] = []- for cells in _split_data_rows(rest, declared):+ for cells in rows:if not cells:continuerecord: dict[str, Any] = {}diff --git a/tests/test_transforms/test_compaction_codec.py b/tests/test_transforms/test_compaction_codec.pyindex fa49077b3..450efe9fe 100644--- a/tests/test_transforms/test_compaction_codec.py+++ b/tests/test_transforms/test_compaction_codec.py@@ -175,6 +175,34 @@ def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}][email protected]("wrapped", [False, True])+ "block",+ [+ "[3]{id:int,name:string}\n1,Alice\n2,Bob\n",+ "[1]{id:int,name:string}\n1,Alice\n2,Bob\n",+ "[1]{id:int}\n",+ '[3]{name:string} __dict:1\n@name=["Alice","Bob"]\n0\n1\n',+ ],+ ids=["missing-row", "extra-row", "empty-body", "factored-missing-row"],+)+def test_row_count_mismatch_is_not_decoded(block: str, wrapped: bool) -> None:+ # A truncated or extended block must not masquerade as the original array.+ assert expand_compacted(json.dumps(block) if wrapped else block) is None+++def test_multiline_cells_count_as_one_row() -> None:+ block = '[2]{id:int,note:string}\n1,"first\nsecond"\n2,last\n'+ assert expand_compacted(block) == [+ {"id": 1, "note": "first\nsecond"},+ {"id": 2, "note": "last"},+ ]+++def test_empty_array_decodes() -> None:+ assert expand_compacted("[0]{id:int}\n") == []++def test_json_column_string_literals_stay_strings_and_values_keep_their_type() -> None:# In a `json` column the native encoder writes a string that looks like a# JSON literal as a quoted JSON string (`"""null"""` is the string `null`),