Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

feat(compress): live-agent densify mode, lossless and cache-safe (#1402)

Tejas Chopra84f0e84dccd3
diff --git a/headroom/__init__.py b/headroom/__init__.py
index 01bc2633b..95a6bdb94 100644
--- a/headroom/__init__.py
+++ b/headroom/__init__.py
@@ -83,7 +83,13 @@ from ._version import __version__ # noqa: F401
# ~microseconds. See `headroom/_ort.py` for the full story.
ensure_ort_dylib_pinned()
-from .compress import CompressConfig, CompressResult, compress, compress_spreadsheet # noqa: E402
+from .compress import ( # noqa: E402
+ CompressConfig,
+ CompressResult,
+ compress,
+ compress_spreadsheet,
+ densify,
+)
# Keep a real callable bound for the one-function compression API so
# `from headroom import compress` is never shadowed by the submodule object.
@@ -175,9 +181,13 @@ __all__ = [
"EmbedderBackend",
# One-function compression API
"compress",
+ "densify",
"compress_spreadsheet",
"CompressConfig",
"CompressResult",
+ # Reverse densified tool output (for consumers of mode="agent" output)
+ "expand_compacted",
+ "is_compacted",
# Hooks
"CompressionHooks",
"CompressContext",
@@ -273,7 +283,10 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {
"reset_otel_metrics": ("headroom.observability", "reset_otel_metrics"),
# One-function API
"compress": ("headroom.compress", "compress"),
+ "densify": ("headroom.compress", "densify"),
"compress_spreadsheet": ("headroom.compress", "compress_spreadsheet"),
+ "expand_compacted": ("headroom.transforms.compaction_codec", "expand_compacted"),
+ "is_compacted": ("headroom.transforms.compaction_codec", "is_compacted"),
# Hooks
"CompressionHooks": ("headroom.hooks", "CompressionHooks"),
"CompressContext": ("headroom.hooks", "CompressContext"),
diff --git a/headroom/compress.py b/headroom/compress.py
index 0923a579c..4fb5f59ac 100644
--- a/headroom/compress.py
+++ b/headroom/compress.py
@@ -73,8 +73,9 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-# Lazy-initialized singleton pipeline
+# Lazy-initialized singleton pipelines (default + agent regime)
_pipeline = None
+_agent_pipeline = None
_pipeline_lock = threading.Lock()
@@ -158,6 +159,26 @@ class CompressConfig:
describing which action was taken (compressed, protected, skipped …) and
how many tokens were spent before/after."""
+ # Regime
+ mode: str | None = None
+ """Compression regime.
+
+ - ``None`` (default): the original behavior — content-aware compression that
+ may use CCR removal (replace-with-marker + retrieve later). Best for a
+ *proxy* compressing conversation history the model has moved past.
+ - ``"agent"``: live-agent mode for compressing tool results an agent will
+ *read back*. Lossless densification only — no CCR/removal, no ML text
+ compression — so it is deterministic and prompt-cache-safe (the same tool
+ output densifies to identical bytes every turn, keeping the cached prefix
+ stable). Forces :attr:`verify_lossless`, so output is never lossy: any
+ message that cannot be proven to round-trip is reverted to its original.
+ Use :func:`densify` as the shorthand."""
+
+ verify_lossless: bool = False
+ """Round-trip each compressed message and revert any that is not provably
+ lossless (keeps the original for that message). Forced on when
+ ``mode='agent'``. Sets :attr:`CompressResult.lossless`."""
+
@dataclass
class CompressResult:
@@ -179,6 +200,14 @@ class CompressResult:
compression_ratio: float = 0.0
transforms_applied: list[str] = field(default_factory=list)
diagnostics: list[MessageDecision] | None = None
+ lossless: bool | None = None
+ """Whether the output is provably lossless. ``True``/``False`` when
+ verification ran (``mode='agent'`` or ``verify_lossless=True``); ``None``
+ when not checked. ``False`` means a removal marker survived but the message
+ could not be reverted (should not happen in ``mode='agent'``)."""
+ reverted_messages: int = 0
+ """Messages reverted to their original form because their compression could
+ not be verified lossless. Only meaningful when verification ran."""
def compress(
@@ -239,7 +268,16 @@ def compress(
collect_diagnostics = cfg.diagnostics or os.environ.get("HEADROOM_DIAGNOSTICS", "") == "1"
- pipeline = _get_pipeline()
+ agent_mode = cfg.mode == "agent"
+ if agent_mode:
+ # Live-agent regime: densify losslessly, never remove. Disable the ML
+ # text compressor so only deterministic, query-independent lossless
+ # transforms run (keeps the cached prefix stable across turns).
+ cfg.kompress_model = "disabled"
+ cfg.verify_lossless = True
+ pipeline = _get_agent_pipeline()
+ else:
+ pipeline = _get_pipeline()
pipeline_extensions = PipelineExtensionManager(hooks=hooks, discover=False)
try:
@@ -271,6 +309,11 @@ def compress(
# alone (position, anomaly) and may drop relevant content.
context = _extract_user_query(messages)
+ # Snapshot the pipeline input so lossless verification can compare each
+ # message against its pre-compression form and revert any that did not
+ # round-trip (mode='agent' / verify_lossless).
+ original_messages = messages
+
result = pipeline.apply(
messages=messages,
model=model,
@@ -310,6 +353,8 @@ def compress(
tokens_saved=0,
compression_ratio=0.0,
transforms_applied=["inflation_guard:reverted"],
+ # Reverting to the originals is trivially lossless.
+ lossless=True if cfg.verify_lossless else None,
)
routing_markers = summarize_routing_markers(result.transforms_applied)
@@ -341,6 +386,28 @@ def compress(
if compressed_event.messages is not None:
compressed_messages = compressed_event.messages
+ # Value-factoring (agent mode): hoist repeated low-cardinality column
+ # values out of densified rows (e.g. file paths in search results).
+ # Lossless + reversible; applied before verification so the round-trip
+ # check validates it too.
+ if agent_mode:
+ compressed_messages = _apply_value_factoring(compressed_messages)
+
+ # Lossless verification: round-trip every compressed message and revert
+ # any that is not provably lossless. Guarantees the output is never
+ # lossy even if a transform slips a removal marker through.
+ lossless: bool | None = None
+ reverted = 0
+ if cfg.verify_lossless:
+ compressed_messages, reverted, lossless = _verify_and_revert(
+ original_messages, compressed_messages
+ )
+
+ # Factoring and reverts change the messages after the pipeline counted
+ # them; recount with the pipeline's own message-counting contract.
+ if agent_mode or cfg.verify_lossless:
+ tokens_after = pipeline._get_tokenizer(model).count_messages(compressed_messages)
+
tokens_saved = tokens_before - tokens_after
ratio = tokens_saved / tokens_before if tokens_before > 0 else 0.0
@@ -366,6 +433,8 @@ def compress(
tokens_saved=tokens_saved,
compression_ratio=ratio,
transforms_applied=result.transforms_applied,
+ lossless=lossless,
+ reverted_messages=reverted,
diagnostics=result.message_decisions if collect_diagnostics else None,
)
@@ -418,6 +487,39 @@ def compress_spreadsheet(
return compress(messages, model=model, model_limit=model_limit, **kwargs)
+def densify(
+ messages: list[dict[str, Any]],
+ model: str = "claude-sonnet-4-5-20250929",
+ model_limit: int = 200000,
+ **kwargs: Any,
+) -> CompressResult:
+ """Lossless, prompt-cache-safe compression for live agents.
+
+ Shorthand for ``compress(messages, mode="agent")``: densifies tool outputs
+ an agent will read back, with **no removal** (no CCR markers, no dropped
+ rows, no ML text compression) and a **verified losslessness guarantee** —
+ any message that cannot be proven to round-trip is reverted to its original.
+ Because only deterministic, query-independent transforms run, the same tool
+ output densifies to identical bytes every turn, so a provider prompt-cache
+ prefix stays stable across the agent loop.
+
+ Inspect :attr:`CompressResult.lossless` (always ``True`` here) and
+ :attr:`CompressResult.reverted_messages` to see how many messages could not
+ be densified losslessly.
+
+ Args:
+ messages: Messages in Anthropic or OpenAI format.
+ model: Model name (token counting / context limit).
+ model_limit: Model context window size in tokens.
+ **kwargs: Forwarded to :func:`compress` (e.g. ``protect_recent``).
+
+ Returns:
+ CompressResult with losslessly densified messages.
+ """
+ kwargs["mode"] = "agent"
+ return compress(messages, model=model, model_limit=model_limit, **kwargs)
+
+
def _get_pipeline() -> Any:
"""Get or create the singleton compression pipeline."""
global _pipeline
@@ -440,3 +542,164 @@ def _get_pipeline() -> Any:
_pipeline = TransformPipeline()
logger.debug("Headroom compression pipeline initialized")
return _pipeline
+
+
+def _get_agent_pipeline() -> Any:
+ """Get or create the singleton live-agent pipeline (lossless densify only).
+
+ A ContentRouter with CCR removal and the ML text compressor disabled, so it
+ only runs deterministic, query-independent lossless transforms (SmartCrusher
+ densification, diff compaction). No CacheAligner stage — agent callers pass
+ the full message list each turn and rely on densification determinism for
+ prompt-cache stability.
+ """
+ global _agent_pipeline
+
+ if _agent_pipeline is not None:
+ return _agent_pipeline
+
+ with _pipeline_lock:
+ if _agent_pipeline is not None:
+ return _agent_pipeline
+
+ from headroom.transforms import TransformPipeline
+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig
+
+ router = ContentRouter(
+ ContentRouterConfig(
+ ccr_enabled=False, # never remove content the agent will read
+ ccr_inject_marker=False, # → emit_opaque_markers off in the engine
+ enable_kompress=False, # lossless only: no ML text compression
+ smart_crusher_with_compaction=True, # lossless-first densification
+ )
+ )
+ _agent_pipeline = TransformPipeline(transforms=[router])
+ logger.debug("Headroom agent (densify-only) pipeline initialized")
+ return _agent_pipeline
+
+
+def _apply_value_factoring(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ """Dictionary-encode low-cardinality columns in every densified string leaf.
+
+ Walks message content and runs :func:`factor_values` on any densified block
+ (no-op for non-densified or already-factored strings). Returns a new list;
+ inputs are not mutated.
+ """
+ from .transforms.compaction_codec import factor_values, is_compacted
+
+ def walk(node: Any) -> Any:
+ if isinstance(node, str):
+ return factor_values(node) if is_compacted(node) else node
+ if isinstance(node, list):
+ return [walk(x) for x in node]
+ if isinstance(node, dict):
+ return {k: walk(v) for k, v in node.items()}
+ return node
+
+ return [walk(m) for m in messages]
+
+
+def _verify_and_revert(
+ original: list[dict[str, Any]],
+ compressed: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], int, bool]:
+ """Revert any compressed message that is not provably lossless.
+
+ Compares each compressed message to its original. A message is kept only if
+ it is unchanged, or every string value it changed is a densified block that
+ round-trips back to the original (and carries no removal marker). Anything
+ unverifiable is reverted to the original — so the result is never lossy.
+
+ Returns ``(messages, num_reverted, lossless)`` where ``lossless`` is True
+ when the final message list contains no surviving removal markers.
+ """
+ from .transforms.compaction_codec import contains_removal_marker
+
+ if len(original) != len(compressed):
+ # Structure changed unexpectedly — fall back to the originals.
+ return list(original), len(compressed), True
+
+ out: list[dict[str, Any]] = []
+ reverted = 0
+ for orig, comp in zip(original, compressed):
+ if orig == comp or _message_is_lossless(orig, comp):
+ out.append(comp)
+ else:
+ out.append(orig)
+ reverted += 1
+
+ lossless = not any(contains_removal_marker(str(m.get("content", ""))) for m in out)
+ return out, reverted, lossless
+
+
+def _message_is_lossless(orig: dict[str, Any], comp: dict[str, Any]) -> bool:
+ """True if ``comp`` is a lossless densification of ``orig``.
+
+ Walks both messages in parallel collecting (original, compressed) string
+ pairs at matching positions; each changed pair must be a densified block
+ that expands back to the original JSON. Any structural divergence or
+ unverifiable change returns False (caller reverts).
+ """
+ pairs: list[tuple[str, str]] = []
+ if not _collect_string_pairs(orig, comp, pairs):
+ return False
+ return all(_string_pair_is_lossless(o, c) for o, c in pairs)
+
+
+def _collect_string_pairs(orig: Any, comp: Any, pairs: list[tuple[str, str]]) -> bool:
+ """Collect aligned string leaves from two parallel structures.
+
+ Returns False if the structures diverge in shape (different types, lengths,
+ or dict keys) — a shape we cannot verify and must conservatively revert.
+ """
+ if isinstance(orig, str) and isinstance(comp, str):
+ if orig != comp:
+ pairs.append((orig, comp))
+ return True
+ if isinstance(orig, dict) and isinstance(comp, dict):
+ if orig.keys() != comp.keys():
+ return False
+ return all(_collect_string_pairs(orig[k], comp[k], pairs) for k in orig)
+ if isinstance(orig, list) and isinstance(comp, list):
+ if len(orig) != len(comp):
+ return False
+ return all(_collect_string_pairs(o, c, pairs) for o, c in zip(orig, comp))
+ # Scalars (int/float/bool/None) must match exactly; transforms only touch
+ # strings, so a changed non-string is unexpected.
+ return bool(orig == comp)
+
+
+def _string_pair_is_lossless(original: str, compressed: str) -> bool:
+ """True if ``compressed`` densifies ``original`` reversibly.
+
+ ``original`` is expected to be a JSON array string; ``compressed`` is the
+ densified block. Lossless iff it carries no removal marker and expands back
+ to the same parsed JSON.
+ """
+ import json as _json
+
+ from .transforms.compaction_codec import (
+ contains_removal_marker,
+ expand_compacted,
+ is_compacted,
+ )
+
+ if contains_removal_marker(compressed):
+ return False
+ if not is_compacted(compressed):
+ return False
+ decoded = expand_compacted(compressed)
+ if decoded is None:
+ return False
+ try:
+ original_obj = _json.loads(original)
+ except (ValueError, TypeError):
+ return False
+ # Compare CANONICAL JSON, not Python ``==``. ``==`` treats True/1/1.0 as
+ # equal, so a bool<->int or int<->float mis-encode would falsely pass this
+ # check — the last line of defense before densified output is trusted as
+ # lossless. ``sort_keys`` keeps the compare insensitive to object key order
+ # while staying type- and value-exact.
+ return _json.dumps(original_obj, sort_keys=True, ensure_ascii=False) == _json.dumps(
+ decoded, sort_keys=True, ensure_ascii=False
+ )
diff --git a/headroom/transforms/compaction_codec.py b/headroom/transforms/compaction_codec.py
new file mode 100644
index 000000000..64cf42c32
--- /dev/null
+++ b/headroom/transforms/compaction_codec.py
@@ -0,0 +1,415 @@
+"""Inverse decoder for SmartCrusher's ``csv-schema`` densification format.
+
+SmartCrusher (the Rust ``compaction`` module) renders a JSON array of objects
+into a compact ``[N]{schema}`` header followed by CSV rows. That direction is
+*encode-only* in the engine — the original is recoverable only by CCR retrieval
+of the stashed blob by hash. For a **live agent** that never wants removal/CCR,
+the densified text must instead be programmatically reversible so callers can:
+
+ 1. **Verify losslessness** — round-trip the densified text and compare it to
+ the original (the guarantee behind ``compress(mode="agent")``).
+ 2. **Re-expand on demand** — a consumer expecting the original ``[{...}]``
+ shape can call :func:`expand_compacted` instead of parsing the dense form.
+
+This module is pure-Python (stdlib only) and deliberately mirrors the wire
+format emitted by ``crates/headroom-core/.../compaction/formatter.rs`` so the two
+stay in lockstep. The cell-encoding contract (post null/empty fix) is:
+
+================== ========================== ============================
+Original value Rendered cell Decoded back to
+================== ========================== ============================
+missing key empty cell key omitted from the dict
+JSON ``null`` ``null`` (bare) ``None``
+``""`` (empty str) ``""`` (quoted empty) ``""``
+literal ``"null"`` ``"null"`` (quoted) ``"null"``
+string w/ ``,"``\\n CSV-quoted the string
+other string bare the string
+number / bool bare ``int``/``float``/``bool``
+nested / array CSV-quoted compact JSON the nested object/array
+opaque blob ``<<ccr:hash,kind,size>>`` NOT reversible from text
+================== ========================== ============================
+
+Schemas declare one column per field as ``name:type`` (or ``name:type?`` when the
+column contains nulls/missing). ``name`` may be dotted (``meta.owner``) when the
+engine flattened a uniform nested object one level; :func:`expand_compacted`
+un-flattens those back into nested dicts.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+# Bare cells the native formatter emits for an absent key and for JSON null.
+# The strings "" and "null" are always CSV-quoted, so neither is ambiguous.
+# Kept in one place so the encoder (Rust) and this decoder agree.
+MISSING_CELL = ""
+NULL_TOKEN = "null"
+
+# Scalar type tags the schema header can declare. ``json`` covers nested
+# objects/arrays and mixed-type columns.
+_SCALAR_TYPES = frozenset({"int", "float", "bool", "string", "json", "null"})
+
+
+def _peel(text: str) -> str:
+ """Strip a JSON-string wrapper around a densified block.
+
+ When ContentRouter densifies a tool result whose content was a JSON array,
+ it re-serializes the compacted block as a JSON string, so the stored content
+ is ``"[N]{...}\\n..."`` (leading quote, escaped newlines) rather than the
+ bare block. Peel that wrapper so the same codec handles both forms.
+ """
+ if text[:1] == '"':
+ try:
+ inner = json.loads(text)
+ except (ValueError, TypeError):
+ return text
+ if isinstance(inner, str):
+ return inner
+ return text
+
+
+def is_compacted(text: str) -> bool:
+ """Return True if ``text`` looks like a ``[N]{schema}`` densified block.
+
+ Structural check (not a heuristic): the first non-empty line must open with
+ ``[<digits>]{`` and the brace section must close on that line. Accepts a
+ JSON-string-wrapped block. Cheap enough to gate ``expand_compacted``.
+ """
+ if not text:
+ return False
+ first = _peel(text).lstrip().split("\n", 1)[0]
+ if not first.startswith("["):
+ return False
+ rbracket = first.find("]")
+ if rbracket < 2 or not first[1:rbracket].isdigit():
+ return False
+ return first[rbracket + 1 : rbracket + 2] == "{" and "}" in first[rbracket:]
+
+
+def contains_removal_marker(text: str) -> bool:
+ """True if the densified text removed content (CCR marker or dropped rows).
+
+ Either signal means the text alone cannot reconstruct the original — the
+ compression was lossy/removal-based, not pure densification.
+ """
+ return "<<ccr:" in text or "__dropped:" in text
+
+
+class _Column:
+ __slots__ = ("name", "type", "nullable", "path")
+
+ def __init__(self, decl: str) -> None:
+ # Split on the final ':' so dotted names (meta.owner) survive.
+ name, _, type_tag = decl.rpartition(":")
+ if not name:
+ name, type_tag = type_tag, "string"
+ self.nullable = type_tag.endswith("?")
+ self.type = type_tag[:-1] if self.nullable else type_tag
+ if self.type not in _SCALAR_TYPES:
+ # Unknown tag → treat as opaque string; keeps the decoder total.
+ self.type = "string"
+ self.name = name
+ self.path = name.split(".")
+
+
+def _parse_header(line: str) -> tuple[list[_Column], int, int] | None:
+ """Parse the header line.
+
+ Form: ``[N]{c:t,c:t?}`` optionally followed by ``__dict:K`` (K value-factor
+ legend lines follow the header) and/or ``__dropped:k``. Returns
+ ``(cols, declared_rows, dict_count)``.
+ """
+ if not line.startswith("["):
+ return None
+ rb = line.find("]")
+ open_brace = line.find("{", rb)
+ close_brace = line.find("}", open_brace)
+ if rb < 0 or open_brace < 0 or close_brace < open_brace:
+ return None
+ try:
+ declared_rows = int(line[1:rb])
+ except ValueError:
+ return None
+ body = line[open_brace + 1 : close_brace]
+ cols = [_Column(d) for d in body.split(",")] if body else []
+ dict_count = 0
+ for token in line[close_brace + 1 :].split():
+ if token.startswith("__dict:"):
+ try:
+ dict_count = int(token[len("__dict:") :])
+ except ValueError:
+ dict_count = 0
+ return cols, declared_rows, dict_count
+
+
+def _iter_rows(blob: str) -> list[list[tuple[bool, str]]]:
+ """Split CSV ``blob`` into rows of ``(was_quoted, value)`` cells.
+
+ Hand-rolled rather than the ``csv`` module because we must preserve the
+ quoted-vs-bare distinction (``"null"`` is a string; bare ``null`` is JSON
+ null) — ``csv.reader`` does not report which cells were quoted.
+ Quotes inside quoted fields are escaped by doubling, matching the Rust
+ ``csv_quote``; newlines inside quotes are honored.
+ """
+ rows: list[list[tuple[bool, str]]] = []
+ row: list[tuple[bool, str]] = []
+ i, n = 0, len(blob)
+ while i < n:
+ if blob[i] == '"': # quoted cell
+ i += 1
+ buf: list[str] = []
+ while i < n:
+ if blob[i] == '"':
+ if i + 1 < n and blob[i + 1] == '"':
+ buf.append('"')
+ i += 2
+ continue
+ i += 1
+ break
+ buf.append(blob[i])
+ i += 1
+ row.append((True, "".join(buf)))
+ if i < n and blob[i] == "\r": # tolerate CRLF row endings
+ i += 1
+ if i < n and blob[i] == ",":
+ i += 1
+ elif i < n and blob[i] == "\n":
+ rows.append(row)
+ row = []
+ i += 1
+ else: # bare cell up to ',' or newline
+ j = i
+ while j < n and blob[j] not in ",\n":
+ j += 1
+ cell = blob[i:j]
+ # A bare cell never legitimately holds '\r' (the formatter quotes
+ # any value containing it), so a trailing '\r' is a CRLF artifact.
+ if cell.endswith("\r"):
+ cell = cell[:-1]
+ row.append((False, cell))
+ if j < n and blob[j] == "\n":
+ rows.append(row)
+ row = []
+ i = j + 1
+ if row:
+ rows.append(row)
+ return rows
+
+
+def _split_data_rows(blob: str, declared: int) -> list[list[tuple[bool, str]]]:
+ """Return the CSV body's data rows, dropping only a trailing-newline artifact.
+
+ The formatter newline-terminates every row, so a single-column row with a
+ missing key renders as a bare empty line — a legitimate ``[(False, "")]``
+ row, NOT an artifact. Only a lone empty cell that EXCEEDS the declared row
+ count is the trailing-newline artifact and is
+ dropped; rows with no cells at all are always dropped.
+ """
+ rows = [r for r in _iter_rows(blob) if r]
+ if declared >= 0 and len(rows) == declared + 1 and rows[-1] == [(False, "")]:
+ rows.pop()
+ return rows
+
+
+def _decode_cell(was_quoted: bool, value: str, col: _Column) -> tuple[bool, Any]:
+ """Return ``(present, decoded)``. ``present=False`` means the key was absent."""
+ if was_quoted:
+ if col.type == "json":
+ try:
+ return True, json.loads(value)
+ except (ValueError, TypeError):
+ return True, value
+ return True, value # quoted "" -> "", quoted "null" -> "null", etc.
+ if value == MISSING_CELL:
+ return False, None
+ if value == NULL_TOKEN:
+ return True, None
+ if col.type == "int":
+ try:
+ return True, int(value)
+ except ValueError:
+ return True, value
+ if col.type == "float":
+ try:
+ return True, float(value)
+ except ValueError:
+ return True, value
+ if col.type == "bool":
+ return True, value == "true"
+ if col.type == "json":
+ try:
+ return True, json.loads(value)
+ except (ValueError, TypeError):
+ return True, value
+ return True, value # string / null / unknown
+
+
+def _render_raw(was_quoted: bool, value: str) -> str:
+ """Reconstruct a cell's exact on-wire text from a parsed ``(quoted, value)``."""
+ if was_quoted:
+ return '"' + value.replace('"', '""') + '"'
+ return value
+
+
+def factor_values(text: str) -> str:
+ """Dictionary-encode low-cardinality string columns. Lossless and reversible.
+
+ SmartCrusher's CSV elides repeated *keys* (the schema header) but still
+ repeats low-cardinality *values* on every row — e.g. a ``search_files``
+ result repeats the full file path once per match. This pass replaces such a
+ column's cells with integer indices into a ``@col=[...]`` legend, hoisting
+ each distinct value out of the row body. A column is encoded only when it
+ *strictly* saves bytes (legend + indices < the repeated cells) — no
+ cardinality threshold, the byte math decides. :func:`expand_compacted`
+ reverses it. Returns ``text`` unchanged when nothing benefits or the input
+ is already factored / not densified.
+
+ Preserves a JSON-string wrapper if present (agent tool results are wrapped).
+ """
+ wrapped = text[:1] == '"'
+ inner = _peel(text)
+ if not is_compacted(inner) or contains_removal_marker(inner):
+ return text
+ head, _, rest = inner.partition("\n")
+ parsed = _parse_header(head.strip())
+ if parsed is None:
+ return text
+ cols, declared, existing_dict = parsed
+ if existing_dict: # already value-factored — idempotent
+ return text
+
+ rows = _split_data_rows(rest, declared)
+ if not rows:
+ return text
+
+ legends: dict[int, list[Any]] = {} # col index -> distinct values (ordered)
+ for j, col in enumerate(cols):
+ if col.type != "string":
+ continue
+ order: list[Any] = []
+ seen: dict[Any, int] = {}
+ raw_bytes = 0
+ index_bytes = 0
+ for cells in rows:
+ if j >= len(cells):
+ continue
+ was_quoted, value = cells[j]
+ if not was_quoted and value in (NULL_TOKEN, MISSING_CELL):
+ continue # null / missing stay inline
+ # _iter_rows already unescaped quoted cells, so the logical string
+ # value is `value` whether or not it was quoted on the wire.
+ decoded = value
+ raw_bytes += len(_render_raw(was_quoted, value))
+ if decoded not in seen:
+ seen[decoded] = len(order)
+ order.append(decoded)
+ index_bytes += len(str(seen[decoded]))
+ if not order:
+ continue
+ legend_json = json.dumps(order, ensure_ascii=False, separators=(",", ":"))
+ legend_bytes = len("@") + len(col.name) + len("=") + len(legend_json) + 1
+ if legend_bytes + index_bytes < raw_bytes:
+ legends[j] = order
+
+ if not legends:
+ return text
+
+ # Rebuild: header + __dict:K, then K legend lines, then re-rendered rows.
+ close = head.find("}", head.find("{"))
+ prefix, trailing = head[: close + 1], head[close + 1 :]
+ new_head = f"{prefix} __dict:{len(legends)}{trailing}"
+
+ legend_lines = []
+ index_maps: dict[int, dict[Any, int]] = {}
+ for j in sorted(legends):
+ order = legends[j]
+ index_maps[j] = {v: i for i, v in enumerate(order)}
+ legend_lines.append(
+ f"@{cols[j].name}=" + json.dumps(order, ensure_ascii=False, separators=(",", ":"))
+ )
+
+ out_rows = []
+ for cells in rows:
+ rendered = []
+ for j in range(len(cols)):
+ if j >= len(cells):
+ break
+ was_quoted, value = cells[j]
+ if j in index_maps and not (not was_quoted and value in (NULL_TOKEN, MISSING_CELL)):
+ rendered.append(str(index_maps[j][value]))
+ else:
+ rendered.append(_render_raw(was_quoted, value))
+ out_rows.append(",".join(rendered))
+
+ factored = "\n".join([new_head, *legend_lines, *out_rows]) + "\n"
+ return json.dumps(factored, ensure_ascii=False) if wrapped else factored
+
+
+def _assign(obj: dict[str, Any], path: list[str], val: Any) -> None:
+ """Set ``val`` at a possibly-dotted ``path``, building nested dicts."""
+ cur = obj
+ for key in path[:-1]:
+ nxt = cur.get(key)
+ if not isinstance(nxt, dict):
+ nxt = {}
+ cur[key] = nxt
+ cur = nxt
+ cur[path[-1]] = val
+
+
+def expand_compacted(text: str) -> list[dict[str, Any]] | None:
+ """Reconstruct the original array of objects from densified ``text``.
+
+ Returns the list of dicts, or ``None`` if ``text`` is not a parseable
+ densified block or carries an irreversible removal marker (CCR / dropped
+ rows) — callers treat ``None`` as "cannot losslessly reverse".
+ """
+ if not is_compacted(text) or contains_removal_marker(text):
+ return None
+ text = _peel(text)
+ head, _, rest = text.partition("\n")
+ parsed = _parse_header(head.strip())
+ if parsed is None:
+ return None
+ cols, declared, dict_count = parsed
+
+ # Value-factor legends: ``__dict:K`` declares K ``@col=[...]`` lines that
+ # map a low-cardinality column's cells (integer indices) back to values.
+ legends: dict[str, list[Any]] = {}
+ for _ in range(dict_count):
+ legend_line, _, rest = rest.partition("\n")
+ name, eq, payload = legend_line.partition("=")
+ if not eq or not name.startswith("@"):
+ return None
+ try:
+ legends[name[1:]] = json.loads(payload)
+ except (ValueError, TypeError):
+ return None
+
+ out: list[dict[str, Any]] = []
+ for cells in _split_data_rows(rest, declared):
+ if not cells:
+ continue
+ record: dict[str, Any] = {}
+ for idx, col in enumerate(cols):
+ if idx >= len(cells):
+ break
+ was_quoted, value = cells[idx]
+ if col.name in legends and not was_quoted:
+ # Dict-encoded cell: null -> None, empty -> missing, else index.
+ if value == MISSING_CELL:
+ continue
+ if value == NULL_TOKEN:
+ _assign(record, col.path, None)
+ continue
+ try:
+ _assign(record, col.path, legends[col.name][int(value)])
+ except (ValueError, IndexError):
+ return None
+ continue
+ present, decoded = _decode_cell(was_quoted, value, col)
+ if present:
+ _assign(record, col.path, decoded)
+ out.append(record)
+ return out
diff --git a/tests/test_densify_agent_mode.py b/tests/test_densify_agent_mode.py
new file mode 100644
index 000000000..4b7d4b229
--- /dev/null
+++ b/tests/test_densify_agent_mode.py
@@ -0,0 +1,218 @@
+"""Live-agent compression mode (``compress(mode='agent')`` / ``densify``).
+
+Agent mode is the densify-only regime for compressing tool results an agent
+reads back: no CCR/removal, no ML text compression, a verified losslessness
+guarantee, and deterministic (prompt-cache-safe) output. These tests assert
+those invariants and that the default ``compress()`` path is unchanged.
+"""
+
+from __future__ import annotations
+
+import json
+
+import headroom
+from headroom.transforms.compaction_codec import expand_compacted, is_compacted
+
+
+def _tool_result_messages(records: list[dict]) -> list[dict]:
+ return [
+ {"role": "user", "content": "run it"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "t1", "name": "q", "input": {}}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "t1", "content": json.dumps(records)}
+ ],
+ },
+ {"role": "user", "content": "summarize"},
+ ]
+
+
+def _records(n: int = 80) -> list[dict]:
+ return [
+ {"id": i, "owner": f"user{i % 5}", "status": "active", "note": f"row padding text {i} here"}
+ for i in range(n)
+ ]
+
+
+def test_densify_compresses_losslessly_without_markers() -> None:
+ res = headroom.densify(_tool_result_messages(_records()))
+ out = json.dumps(res.messages)
+ assert res.tokens_saved > 0
+ assert res.lossless is True
+ assert res.reverted_messages == 0
+ assert "<<ccr:" not in out # no removal markers
+ assert "__dropped:" not in out # no dropped rows
+
+
+def test_densified_tool_result_round_trips() -> None:
+ records = _records()
+ res = headroom.densify(_tool_result_messages(records))
+ content = res.messages[2]["content"][0]["content"]
+ assert is_compacted(content)
+ assert expand_compacted(content) == records
+
+
+def test_opaque_blob_is_preserved_not_pointerized() -> None:
+ blob = "X" * 2000 # well over the 256B opaque threshold
+ msgs = [
+ {"role": "user", "content": "go"},
+ {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": "t2", "name": "read", "input": {}}],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "t2",
+ "content": json.dumps({"file": "x", "body": blob}),
+ }
+ ],
+ },
+ ]
+ res = headroom.densify(msgs)
+ out = json.dumps(res.messages)
+ assert res.lossless is True
+ assert "<<ccr:" not in out
+ assert blob in out # the blob the agent needs is still there verbatim
+
+
+def test_deterministic_output_for_cache_safety() -> None:
+ msgs = _tool_result_messages(_records())
+ a = headroom.densify(msgs)
+ b = headroom.densify(_tool_result_messages(_records()))
+ assert json.dumps(a.messages) == json.dumps(b.messages)
+
+
+def test_value_factoring_applied_in_agent_mode() -> None:
+ paths = ["./pkg/module_one.py", "./pkg/module_two.py", "./pkg/module_three.py"]
+ records = [
+ {"path": paths[i % len(paths)], "line": 10 + i, "content": f"call_{i}()"} for i in range(40)
+ ]
+ res = headroom.densify(_tool_result_messages(records))
+ content = res.messages[2]["content"][0]["content"]
+ assert "__dict:" in content # legend present
+ assert expand_compacted(content) == records # lossless
+
+
+def test_default_compress_is_unchanged() -> None:
+ res = headroom.compress(_tool_result_messages(_records()), kompress_model="disabled")
+ # Default regime does not run lossless verification.
+ assert res.lossless is None
+ assert res.reverted_messages == 0
+ assert res.tokens_saved > 0
+
+
+def test_verify_lossless_flag_without_agent_mode() -> None:
+ res = headroom.compress(
+ _tool_result_messages(_records()), kompress_model="disabled", verify_lossless=True
+ )
+ assert res.lossless is True
+
+
+def test_empty_and_passthrough_inputs() -> None:
+ assert headroom.densify([]).messages == []
+ short = [{"role": "user", "content": "hi"}]
+ assert headroom.densify(short).messages == short
+
+
+def test_openai_format_tool_message_is_densified_losslessly() -> None:
+ # Generic harness: OpenAI chat format (role='tool', string content) — not
+ # Anthropic content blocks. Agent mode must handle it the same way.
+ records = [{"id": i, "tag": f"t{i % 4}", "note": f"value padding text {i}"} for i in range(60)]
+ msgs = [
+ {"role": "user", "content": "do it"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {"id": "c1", "type": "function", "function": {"name": "q", "arguments": "{}"}}
+ ],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": json.dumps(records)},
+ ]
+ res = headroom.densify(msgs)
+ assert res.lossless is True
+ assert res.tokens_saved > 0
+ content = res.messages[2]["content"]
+ assert is_compacted(content)
+ assert expand_compacted(content) == records
+
+
+def test_reverse_helpers_are_public() -> None:
+ # A consumer of densified output must be able to reverse it from the
+ # top-level package, not only a deep submodule path.
+ from headroom import expand_compacted as top_expand
+ from headroom import is_compacted as top_is
+
+ records = [{"id": i, "tag": f"t{i % 3}"} for i in range(30)]
+ res = headroom.densify(_tool_result_messages(records))
+ content = res.messages[2]["content"][0]["content"]
+ assert top_is(content)
+ assert top_expand(content) == records
+
+
+def test_verify_rejects_bool_int_conflation() -> None:
+ # The losslessness check compares CANONICAL JSON, not Python `==` (which
+ # treats True/1/1.0 as equal). A densified block that decodes a bool to an
+ # int must NOT be accepted as lossless — else a type mis-encode passes the
+ # last line of defense before output is trusted.
+ from headroom.compress import _string_pair_is_lossless
+
+ # `[1]{a:int}\n1\n` decodes to [{"a": 1}] — an int, not the original bool.
+ assert _string_pair_is_lossless('[{"a":true}]', "[1]{a:int}\n1\n") is False
+ # positive control: a faithful int round-trip is still accepted.
+ assert _string_pair_is_lossless('[{"a":1}]', "[1]{a:int}\n1\n") is True
+
+
+def _count_messages(messages: list[dict]) -> int:
+ from headroom.compress import _get_agent_pipeline
+
+ return (
+ _get_agent_pipeline()._get_tokenizer("claude-sonnet-4-5-20250929").count_messages(messages)
+ )
+
+
+def _repeated_path_records() -> list[dict]:
+ paths = [f"./src/very/long/package/path/module_{k}/implementation_file.py" for k in range(3)]
+ return [{"path": paths[i % 3], "line": i, "text": f"hit {i}"} for i in range(40)]
+
+
+def test_tokens_after_counts_value_factored_messages() -> None:
+ res = headroom.densify(_tool_result_messages(_repeated_path_records()))
+ assert "__dict:" in res.messages[2]["content"][0]["content"]
+ assert res.reverted_messages == 0
+ assert res.tokens_after == _count_messages(res.messages)
+ assert res.tokens_saved == res.tokens_before - res.tokens_after
+
+
+def test_tokens_after_counts_reverted_messages(monkeypatch) -> None:
+ import importlib
+
+ # `headroom.compress` the attribute is the function; fetch the module.
+ compress_module = importlib.import_module("headroom.compress")
+ monkeypatch.setattr(compress_module, "_message_is_lossless", lambda orig, comp: False)
+ msgs = _tool_result_messages(_repeated_path_records())
+ res = headroom.densify(msgs)
+ assert res.reverted_messages == 1
+ assert res.messages == msgs
+ assert res.tokens_after == _count_messages(msgs)
+ assert res.tokens_saved == 0
+
+
+def test_new_fields_do_not_shift_positional_arguments() -> None:
+ # New fields go after `diagnostics` so existing positional construction
+ # still binds `diagnostics` where it did before this mode existed.
+ from dataclasses import fields
+
+ from headroom.compress import CompressConfig, CompressResult
+
+ config_names = [f.name for f in fields(CompressConfig)]
+ assert config_names[-3:] == ["diagnostics", "mode", "verify_lossless"]
+ result_names = [f.name for f in fields(CompressResult)]
+ assert result_names[-3:] == ["diagnostics", "lossless", "reverted_messages"]
diff --git a/tests/test_transforms/test_compaction_codec.py b/tests/test_transforms/test_compaction_codec.py
new file mode 100644
index 000000000..67d100f56
--- /dev/null
+++ b/tests/test_transforms/test_compaction_codec.py
@@ -0,0 +1,180 @@
+"""Inverse decoder + value-factoring for the ``csv-schema`` densified format.
+
+These exercise the Python codec (``headroom.transforms.compaction_codec``)
+against *real* Rust SmartCrusher output so the encoder and decoder stay in
+lockstep:
+
+- losslessness of densification is programmatically verifiable (round-trip);
+- the null / empty-string / missing-key / literal ``"null"`` distinctions
+ survive, matching the native formatter's encoding;
+- value-factoring dictionary-encodes low-cardinality columns reversibly and
+ only when it saves bytes.
+"""
+
+from __future__ import annotations
+
+import json
+
+from headroom.transforms.compaction_codec import (
+ contains_removal_marker,
+ expand_compacted,
+ factor_values,
+ is_compacted,
+)
+from headroom.transforms.smart_crusher import SmartCrusher
+
+
+def _crush(records: list[dict]) -> str:
+ return SmartCrusher().crush_array_json(json.dumps(records)).get("compacted") or ""
+
+
+def test_round_trips_real_rust_output() -> None:
+ records = [
+ {"id": i, "owner": f"user{i % 5}", "score": i / 3, "active": i % 2 == 0} for i in range(40)
+ ]
+ comp = _crush(records)
+ assert is_compacted(comp)
+ assert expand_compacted(comp) == records
+
+
+def test_null_empty_missing_and_literal_backslash_n_are_distinct() -> None:
+ records: list[dict] = []
+ for i in range(40):
+ if i == 0:
+ records.append({"a": i, "b": i, "c": None}) # null
+ elif i == 1:
+ records.append({"a": i, "b": i, "c": ""}) # empty string
+ elif i == 2:
+ records.append({"a": i, "b": i}) # missing key
+ elif i == 3:
+ records.append({"a": i, "b": i, "c": "\\N"}) # literal backslash-N
+ elif i == 4:
+ records.append({"a": i, "b": i, "c": "null"}) # literal string "null"
+ else:
+ records.append({"a": i, "b": i, "c": f"v{i}"})
+ comp = _crush(records)
+ decoded = expand_compacted(comp)
+ assert decoded is not None
+ assert "c" not in decoded[2] # missing stays absent
+ assert decoded[0]["c"] is None # null
+ assert decoded[1]["c"] == "" # empty string
+ assert decoded[3]["c"] == "\\N" # literal preserved
+ assert decoded[4]["c"] == "null" # quoted literal is a string, not null
+ assert decoded == records
+
+
+def test_strings_with_commas_quotes_and_unicode() -> None:
+ records = [
+ {"k": i, "v": v}
+ for i, v in enumerate(
+ ["a,b", 'he said "hi"', "café 日本語 🔥", "trailing space ", "", "line1"]
+ )
+ ]
+ # pad so the array clears the densification token gate
+ records += [{"k": 100 + i, "v": f"pad value {i}"} for i in range(30)]
+ comp = _crush(records)
+ assert expand_compacted(comp) == records
+
+
+def test_nested_object_flattening_round_trips() -> None:
+ records = [{"id": i, "meta": {"owner": f"u{i % 3}", "rank": i}} for i in range(30)]
+ comp = _crush(records)
+ decoded = expand_compacted(comp)
+ assert decoded == records
+
+
+def test_value_factoring_beats_plain_on_repeated_paths() -> None:
+ paths = ["./a/very/long/path/one.py", "./another/long/path/two.py", "./third.py"]
+ records = [
+ {"path": paths[i % len(paths)], "line": 100 + i, "content": f"def f_{i}(): pass"}
+ for i in range(40)
+ ]
+ plain = _crush(records)
+ factored = factor_values(plain)
+ assert "__dict:" in factored
+ assert len(factored) < len(plain) # strictly smaller
+ assert expand_compacted(factored) == records # still lossless
+
+
+def test_value_factoring_is_idempotent() -> None:
+ records = [{"path": f"./p{i % 3}.py", "line": i, "content": f"x{i}"} for i in range(30)]
+ factored = factor_values(_crush(records))
+ assert factor_values(factored) == factored
+
+
+def test_value_factoring_noop_when_no_benefit() -> None:
+ # All-distinct high-entropy column → no repetition to hoist.
+ records = [{"id": i, "uuid": f"id-{i}-{'x' * 20}"} for i in range(30)]
+ plain = _crush(records)
+ assert factor_values(plain) == plain
+
+
+def test_value_factoring_preserves_nulls_in_dict_column() -> None:
+ records = [
+ {"path": (None if i % 7 == 0 else f"./p{i % 3}.py"), "line": i, "content": f"c{i}"}
+ for i in range(40)
+ ]
+ factored = factor_values(_crush(records))
+ assert expand_compacted(factored) == records
+
+
+def test_removal_marker_makes_text_unreversible() -> None:
+ text = "[5]{a:int,blob:string}\n0,<<ccr:abc123,string,1.2KB>>\n"
+ assert contains_removal_marker(text)
+ assert expand_compacted(text) is None
+
+
+def test_json_string_wrapped_block_is_recognized() -> None:
+ records = [{"id": i, "tag": f"t{i % 4}"} for i in range(30)]
+ bare = _crush(records)
+ wrapped = json.dumps(bare) # the form ContentRouter stores for tool results
+ assert is_compacted(wrapped)
+ assert expand_compacted(wrapped) == records
+
+
+def test_non_compacted_text_is_ignored() -> None:
+ assert not is_compacted("just some prose, not densified")
+ assert expand_compacted("just some prose") is None
+ assert factor_values("just some prose") == "just some prose"
+
+
+def test_crlf_line_endings_round_trip() -> None:
+ # A harness that round-trips the block through a Windows text-mode file (or
+ # otherwise normalizes line endings) must still decode. The format is
+ # LF-internal; the decoder tolerates CRLF defensively.
+ records = [{"id": i, "name": f"n{i % 4}", "v": f"val {i}"} for i in range(30)]
+ bare = _crush(records)
+ crlf = bare.replace("\n", "\r\n")
+ assert expand_compacted(crlf) == records
+ assert expand_compacted(crlf) == expand_compacted(bare)
+
+
+def test_value_factoring_handles_comma_and_quote_values() -> None:
+ # Repeated values that need CSV-quoting must still factor losslessly.
+ vals = ["a, b.py", 'has "quotes"', "c/d.py"]
+ records = [{"path": vals[i % len(vals)], "line": i, "content": f"x{i}"} for i in range(30)]
+ factored = factor_values(_crush(records))
+ assert "__dict:" in factored
+ assert expand_compacted(factored) == records
+
+
+def test_single_column_missing_row_is_not_dropped() -> None:
+ # Native encoding: missing key is a bare empty cell, null is bare `null`,
+ # and "" / "null" are quoted. A trailing single-column missing row renders
+ # as a bare empty line; the declared row count keeps it from being dropped
+ # as a trailing-newline artifact.
+ block = '[4]{k:string?}\n""\nnull\n"null"\n\n'
+ assert expand_compacted(block) == [{"k": ""}, {"k": None}, {"k": "null"}, {}]
+
+
+def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:
+ # A blank line BEYOND the declared row count is the trailing-newline
+ # artifact and must be dropped (not decoded as a spurious extra null row).
+ assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}]
+
+
+def test_single_column_null_survives_value_factoring() -> None:
+ block = "[3]{k:string?}\nx\ny\nnull\n"
+ factored = factor_values(block)
+ assert expand_compacted(factored) == expand_compacted(block)
+ assert expand_compacted(factored) == [{"k": "x"}, {"k": "y"}, {"k": None}]