Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
"""Inverse decoder + value-factoring for the ``csv-schema`` densified format.
These exercise the Python codec (``headroom.transforms.compaction_codec``)
against *real* Rust SmartCrusher output so the encoder and decoder stay in
lockstep:
- losslessness of densification is programmatically verifiable (round-trip);
- the null / empty-string / missing-key / literal ``"null"`` distinctions
survive, matching the native formatter's encoding;
- value-factoring dictionary-encodes low-cardinality columns reversibly and
only when it saves bytes.
"""
from __future__ import annotations
import json
import pytest
from headroom.transforms.compaction_codec import (
contains_removal_marker,
expand_compacted,
factor_values,
is_compacted,
)
from headroom.transforms.smart_crusher import SmartCrusher
def _crush(records: list[dict]) -> str:
return SmartCrusher().crush_array_json(json.dumps(records)).get("compacted") or ""
def test_round_trips_real_rust_output() -> None:
records = [
{"id": i, "owner": f"user{i % 5}", "score": i / 3, "active": i % 2 == 0} for i in range(40)
]
comp = _crush(records)
assert is_compacted(comp)
assert expand_compacted(comp) == records
def test_null_empty_missing_and_literal_backslash_n_are_distinct() -> None:
records: list[dict] = []
for i in range(40):
if i == 0:
records.append({"a": i, "b": i, "c": None}) # null
elif i == 1:
records.append({"a": i, "b": i, "c": ""}) # empty string
elif i == 2:
records.append({"a": i, "b": i}) # missing key
elif i == 3:
records.append({"a": i, "b": i, "c": "\\N"}) # literal backslash-N
elif i == 4:
records.append({"a": i, "b": i, "c": "null"}) # literal string "null"
else:
records.append({"a": i, "b": i, "c": f"v{i}"})
comp = _crush(records)
decoded = expand_compacted(comp)
assert decoded is not None
assert "c" not in decoded[2] # missing stays absent
assert decoded[0]["c"] is None # null
assert decoded[1]["c"] == "" # empty string
assert decoded[3]["c"] == "\\N" # literal preserved
assert decoded[4]["c"] == "null" # quoted literal is a string, not null
assert decoded == records
def test_strings_with_commas_quotes_and_unicode() -> None:
records = [
{"k": i, "v": v}
for i, v in enumerate(
["a,b", 'he said "hi"', "café 日本語 🔥", "trailing space ", "", "line1"]
)
]
# pad so the array clears the densification token gate
records += [{"k": 100 + i, "v": f"pad value {i}"} for i in range(30)]
comp = _crush(records)
assert expand_compacted(comp) == records
def test_nested_object_flattening_round_trips() -> None:
records = [{"id": i, "meta": {"owner": f"u{i % 3}", "rank": i}} for i in range(30)]
comp = _crush(records)
decoded = expand_compacted(comp)
assert decoded == records
def test_value_factoring_beats_plain_on_repeated_paths() -> None:
paths = ["./a/very/long/path/one.py", "./another/long/path/two.py", "./third.py"]
records = [
{"path": paths[i % len(paths)], "line": 100 + i, "content": f"def f_{i}(): pass"}
for i in range(40)
]
plain = _crush(records)
factored = factor_values(plain)
assert "__dict:" in factored
assert len(factored) < len(plain) # strictly smaller
assert expand_compacted(factored) == records # still lossless
def test_value_factoring_is_idempotent() -> None:
records = [{"path": f"./p{i % 3}.py", "line": i, "content": f"x{i}"} for i in range(30)]
factored = factor_values(_crush(records))
assert factor_values(factored) == factored
def test_value_factoring_noop_when_no_benefit() -> None:
# All-distinct high-entropy column → no repetition to hoist.
records = [{"id": i, "uuid": f"id-{i}-{'x' * 20}"} for i in range(30)]
plain = _crush(records)
assert factor_values(plain) == plain
def test_value_factoring_preserves_nulls_in_dict_column() -> None:
records = [
{"path": (None if i % 7 == 0 else f"./p{i % 3}.py"), "line": i, "content": f"c{i}"}
for i in range(40)
]
factored = factor_values(_crush(records))
assert expand_compacted(factored) == records
def test_removal_marker_makes_text_unreversible() -> None:
text = "[5]{a:int,blob:string}\n0,<<ccr:abc123,string,1.2KB>>\n"
assert contains_removal_marker(text)
assert expand_compacted(text) is None
def test_json_string_wrapped_block_is_recognized() -> None:
records = [{"id": i, "tag": f"t{i % 4}"} for i in range(30)]
bare = _crush(records)
wrapped = json.dumps(bare) # the form ContentRouter stores for tool results
assert is_compacted(wrapped)
assert expand_compacted(wrapped) == records
def test_non_compacted_text_is_ignored() -> None:
assert not is_compacted("just some prose, not densified")
assert expand_compacted("just some prose") is None
assert factor_values("just some prose") == "just some prose"
def test_crlf_line_endings_round_trip() -> None:
# A harness that round-trips the block through a Windows text-mode file (or
# otherwise normalizes line endings) must still decode. The format is
# LF-internal; the decoder tolerates CRLF defensively.
records = [{"id": i, "name": f"n{i % 4}", "v": f"val {i}"} for i in range(30)]
bare = _crush(records)
crlf = bare.replace("\n", "\r\n")
assert expand_compacted(crlf) == records
assert expand_compacted(crlf) == expand_compacted(bare)
def test_value_factoring_handles_comma_and_quote_values() -> None:
# Repeated values that need CSV-quoting must still factor losslessly.
vals = ["a, b.py", 'has "quotes"', "c/d.py"]
records = [{"path": vals[i % len(vals)], "line": i, "content": f"x{i}"} for i in range(30)]
factored = factor_values(_crush(records))
assert "__dict:" in factored
assert expand_compacted(factored) == records
def test_single_column_missing_row_is_not_dropped() -> None:
# Native encoding: missing key is a bare empty cell, null is bare `null`,
# and "" / "null" are quoted. A trailing single-column missing row renders
# as a bare empty line; the declared row count keeps it from being dropped
# as a trailing-newline artifact.
block = '[4]{k:string?}\n""\nnull\n"null"\n\n'
assert expand_compacted(block) == [{"k": ""}, {"k": None}, {"k": "null"}, {}]
def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:
# A blank line BEYOND the declared row count is the trailing-newline
# artifact and must be dropped (not decoded as a spurious extra null row).
assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}]
@pytest.mark.parametrize("wrapped", [False, True])
@pytest.mark.parametrize(
"block",
[
"[3]{id:int,name:string}\n1,Alice\n2,Bob\n",
"[1]{id:int,name:string}\n1,Alice\n2,Bob\n",
"[1]{id:int}\n",
'[3]{name:string} __dict:1\n@name=["Alice","Bob"]\n0\n1\n',
],
ids=["missing-row", "extra-row", "empty-body", "factored-missing-row"],
)
def test_row_count_mismatch_is_not_decoded(block: str, wrapped: bool) -> None:
# A truncated or extended block must not masquerade as the original array.
assert expand_compacted(json.dumps(block) if wrapped else block) is None
def test_multiline_cells_count_as_one_row() -> None:
block = '[2]{id:int,note:string}\n1,"first\nsecond"\n2,last\n'
assert expand_compacted(block) == [
{"id": 1, "note": "first\nsecond"},
{"id": 2, "note": "last"},
]
def test_empty_array_decodes() -> None:
assert expand_compacted("[0]{id:int}\n") == []
def test_json_column_string_literals_stay_strings_and_values_keep_their_type() -> None:
# In a `json` column the native encoder writes a string that looks like a
# JSON literal as a quoted JSON string (`"""null"""` is the string `null`),
# while real values stay bare. The decoder must tell the two apart.
block = (
"[9]{id:int,v:json?}\n"
'1,"""null"""\n'
"2,null\n"
'3,"""true"""\n'
"4,true\n"
'5,"""1"""\n'
"6,1\n"
'7,"""NaN"""\n'
'8,"""[1]"""\n'
"9,hello\n"
)
decoded = expand_compacted(block)
assert decoded is not None
values = [row["v"] for row in decoded]
assert values == ["null", None, "true", True, "1", 1, "NaN", "[1]", "hello"]
assert [type(v) for v in values] == [str, type(None), str, bool, str, int, str, str, str]
def test_single_column_null_survives_value_factoring() -> None:
block = "[3]{k:string?}\nx\ny\nnull\n"
factored = factor_values(block)
assert expand_compacted(factored) == expand_compacted(block)
assert expand_compacted(factored) == [{"k": "x"}, {"k": "y"}, {"k": None}]
def test_negative_dictionary_index_is_rejected() -> None:
# Python's -1 would silently pick the last legend value.
assert expand_compacted('[1]{tag:string} __dict:1\n@tag=["first","last"]\n-1\n') is None
@pytest.mark.parametrize("legend", ["42", '"ab"', '{"a":1}', "null"])
def test_non_array_legend_is_rejected(legend: str) -> None:
assert expand_compacted(f"[1]{{tag:string}} __dict:1\n@tag={legend}\n0\n") is None
def test_dictionary_cells_still_decode_edge_values() -> None:
block = '[5]{tag:string?} __dict:1\n@tag=["first","last"]\n0\n1\n"-1"\nnull\n\n'
assert expand_compacted(block) == [
{"tag": "first"},
{"tag": "last"},
{"tag": "-1"},
{"tag": None},
{},
]