Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(codec): reject densified blocks with mismatched row counts (#4271)

Andrew Barnes75df01d627cf
diff --git a/headroom/transforms/compaction_codec.py b/headroom/transforms/compaction_codec.py
index 5413cb7a8..41933630d 100644
--- a/headroom/transforms/compaction_codec.py
+++ b/headroom/transforms/compaction_codec.py
@@ -362,8 +362,9 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
"""Reconstruct the original array of objects from densified ``text``.
Returns the list of dicts, or ``None`` if ``text`` is not a parseable
- densified block or carries an irreversible removal marker (CCR / dropped
- rows) — callers treat ``None`` as "cannot losslessly reverse".
+ densified block, its row count differs from the header, or it carries an
+ irreversible removal marker (CCR / dropped rows). Callers treat ``None``
+ as "cannot losslessly reverse".
"""
if not is_compacted(text) or contains_removal_marker(text):
return None
@@ -390,8 +391,12 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
return None
legends[name[1:]] = result
+ rows = _split_data_rows(rest, declared)
+ if len(rows) != declared:
+ return None
+
out: list[dict[str, Any]] = []
- for cells in _split_data_rows(rest, declared):
+ for cells in rows:
if not cells:
continue
record: dict[str, Any] = {}
diff --git a/tests/test_transforms/test_compaction_codec.py b/tests/test_transforms/test_compaction_codec.py
index fa49077b3..450efe9fe 100644
--- a/tests/test_transforms/test_compaction_codec.py
+++ b/tests/test_transforms/test_compaction_codec.py
@@ -175,6 +175,34 @@ def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:
assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}]
[email protected]("wrapped", [False, True])
+ "block",
+ [
+ "[3]{id:int,name:string}\n1,Alice\n2,Bob\n",
+ "[1]{id:int,name:string}\n1,Alice\n2,Bob\n",
+ "[1]{id:int}\n",
+ '[3]{name:string} __dict:1\n@name=["Alice","Bob"]\n0\n1\n',
+ ],
+ ids=["missing-row", "extra-row", "empty-body", "factored-missing-row"],
+)
+def test_row_count_mismatch_is_not_decoded(block: str, wrapped: bool) -> None:
+ # A truncated or extended block must not masquerade as the original array.
+ assert expand_compacted(json.dumps(block) if wrapped else block) is None
+
+
+def test_multiline_cells_count_as_one_row() -> None:
+ block = '[2]{id:int,note:string}\n1,"first\nsecond"\n2,last\n'
+ assert expand_compacted(block) == [
+ {"id": 1, "note": "first\nsecond"},
+ {"id": 2, "note": "last"},
+ ]
+
+
+def test_empty_array_decodes() -> None:
+ assert expand_compacted("[0]{id:int}\n") == []
+
+
def test_json_column_string_literals_stay_strings_and_values_keep_their_type() -> None:
# In a `json` column the native encoder writes a string that looks like a
# JSON literal as a quoted JSON string (`"""null"""` is the string `null`),