Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(compress): live-agent densify mode, lossless and cache-safe (#1402)
Tejas Chopra84f0e84dccd3
diff --git a/headroom/__init__.py b/headroom/__init__.pyindex 01bc2633b..95a6bdb94 100644--- a/headroom/__init__.py+++ b/headroom/__init__.py@@ -83,7 +83,13 @@ from ._version import __version__ # noqa: F401# ~microseconds. See `headroom/_ort.py` for the full story.ensure_ort_dylib_pinned()-from .compress import CompressConfig, CompressResult, compress, compress_spreadsheet # noqa: E402+from .compress import ( # noqa: E402+ CompressConfig,+ CompressResult,+ compress,+ compress_spreadsheet,+ densify,+)# Keep a real callable bound for the one-function compression API so# `from headroom import compress` is never shadowed by the submodule object.@@ -175,9 +181,13 @@ __all__ = ["EmbedderBackend",# One-function compression API"compress",+ "densify","compress_spreadsheet","CompressConfig","CompressResult",+ # Reverse densified tool output (for consumers of mode="agent" output)+ "expand_compacted",+ "is_compacted",# Hooks"CompressionHooks","CompressContext",@@ -273,7 +283,10 @@ _LAZY_EXPORTS: dict[str, tuple[str, str]] = {"reset_otel_metrics": ("headroom.observability", "reset_otel_metrics"),# One-function API"compress": ("headroom.compress", "compress"),+ "densify": ("headroom.compress", "densify"),"compress_spreadsheet": ("headroom.compress", "compress_spreadsheet"),+ "expand_compacted": ("headroom.transforms.compaction_codec", "expand_compacted"),+ "is_compacted": ("headroom.transforms.compaction_codec", "is_compacted"),# Hooks"CompressionHooks": ("headroom.hooks", "CompressionHooks"),"CompressContext": ("headroom.hooks", "CompressContext"),diff --git a/headroom/compress.py b/headroom/compress.pyindex 0923a579c..4fb5f59ac 100644--- a/headroom/compress.py+++ b/headroom/compress.py@@ -73,8 +73,9 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)-# Lazy-initialized singleton pipeline+# Lazy-initialized singleton pipelines (default + agent regime)_pipeline = None+_agent_pipeline = None_pipeline_lock = threading.Lock()@@ -158,6 +159,26 @@ class CompressConfig:describing which action was taken (compressed, protected, skipped …) andhow many tokens were spent before/after."""+ # Regime+ mode: str | None = None+ """Compression regime.++ - ``None`` (default): the original behavior — content-aware compression that+ may use CCR removal (replace-with-marker + retrieve later). Best for a+ *proxy* compressing conversation history the model has moved past.+ - ``"agent"``: live-agent mode for compressing tool results an agent will+ *read back*. Lossless densification only — no CCR/removal, no ML text+ compression — so it is deterministic and prompt-cache-safe (the same tool+ output densifies to identical bytes every turn, keeping the cached prefix+ stable). Forces :attr:`verify_lossless`, so output is never lossy: any+ message that cannot be proven to round-trip is reverted to its original.+ Use :func:`densify` as the shorthand."""++ verify_lossless: bool = False+ """Round-trip each compressed message and revert any that is not provably+ lossless (keeps the original for that message). Forced on when+ ``mode='agent'``. Sets :attr:`CompressResult.lossless`."""+@dataclassclass CompressResult:@@ -179,6 +200,14 @@ class CompressResult:compression_ratio: float = 0.0transforms_applied: list[str] = field(default_factory=list)diagnostics: list[MessageDecision] | None = None+ lossless: bool | None = None+ """Whether the output is provably lossless. ``True``/``False`` when+ verification ran (``mode='agent'`` or ``verify_lossless=True``); ``None``+ when not checked. ``False`` means a removal marker survived but the message+ could not be reverted (should not happen in ``mode='agent'``)."""+ reverted_messages: int = 0+ """Messages reverted to their original form because their compression could+ not be verified lossless. Only meaningful when verification ran."""def compress(@@ -239,7 +268,16 @@ def compress(collect_diagnostics = cfg.diagnostics or os.environ.get("HEADROOM_DIAGNOSTICS", "") == "1"- pipeline = _get_pipeline()+ agent_mode = cfg.mode == "agent"+ if agent_mode:+ # Live-agent regime: densify losslessly, never remove. Disable the ML+ # text compressor so only deterministic, query-independent lossless+ # transforms run (keeps the cached prefix stable across turns).+ cfg.kompress_model = "disabled"+ cfg.verify_lossless = True+ pipeline = _get_agent_pipeline()+ else:+ pipeline = _get_pipeline()pipeline_extensions = PipelineExtensionManager(hooks=hooks, discover=False)try:@@ -271,6 +309,11 @@ def compress(# alone (position, anomaly) and may drop relevant content.context = _extract_user_query(messages)+ # Snapshot the pipeline input so lossless verification can compare each+ # message against its pre-compression form and revert any that did not+ # round-trip (mode='agent' / verify_lossless).+ original_messages = messages+result = pipeline.apply(messages=messages,model=model,@@ -310,6 +353,8 @@ def compress(tokens_saved=0,compression_ratio=0.0,transforms_applied=["inflation_guard:reverted"],+ # Reverting to the originals is trivially lossless.+ lossless=True if cfg.verify_lossless else None,)routing_markers = summarize_routing_markers(result.transforms_applied)@@ -341,6 +386,28 @@ def compress(if compressed_event.messages is not None:compressed_messages = compressed_event.messages+ # Value-factoring (agent mode): hoist repeated low-cardinality column+ # values out of densified rows (e.g. file paths in search results).+ # Lossless + reversible; applied before verification so the round-trip+ # check validates it too.+ if agent_mode:+ compressed_messages = _apply_value_factoring(compressed_messages)++ # Lossless verification: round-trip every compressed message and revert+ # any that is not provably lossless. Guarantees the output is never+ # lossy even if a transform slips a removal marker through.+ lossless: bool | None = None+ reverted = 0+ if cfg.verify_lossless:+ compressed_messages, reverted, lossless = _verify_and_revert(+ original_messages, compressed_messages+ )++ # Factoring and reverts change the messages after the pipeline counted+ # them; recount with the pipeline's own message-counting contract.+ if agent_mode or cfg.verify_lossless:+ tokens_after = pipeline._get_tokenizer(model).count_messages(compressed_messages)+tokens_saved = tokens_before - tokens_afterratio = tokens_saved / tokens_before if tokens_before > 0 else 0.0@@ -366,6 +433,8 @@ def compress(tokens_saved=tokens_saved,compression_ratio=ratio,transforms_applied=result.transforms_applied,+ lossless=lossless,+ reverted_messages=reverted,diagnostics=result.message_decisions if collect_diagnostics else None,)@@ -418,6 +487,39 @@ def compress_spreadsheet(return compress(messages, model=model, model_limit=model_limit, **kwargs)+def densify(+ messages: list[dict[str, Any]],+ model: str = "claude-sonnet-4-5-20250929",+ model_limit: int = 200000,+ **kwargs: Any,+) -> CompressResult:+ """Lossless, prompt-cache-safe compression for live agents.++ Shorthand for ``compress(messages, mode="agent")``: densifies tool outputs+ an agent will read back, with **no removal** (no CCR markers, no dropped+ rows, no ML text compression) and a **verified losslessness guarantee** —+ any message that cannot be proven to round-trip is reverted to its original.+ Because only deterministic, query-independent transforms run, the same tool+ output densifies to identical bytes every turn, so a provider prompt-cache+ prefix stays stable across the agent loop.++ Inspect :attr:`CompressResult.lossless` (always ``True`` here) and+ :attr:`CompressResult.reverted_messages` to see how many messages could not+ be densified losslessly.++ Args:+ messages: Messages in Anthropic or OpenAI format.+ model: Model name (token counting / context limit).+ model_limit: Model context window size in tokens.+ **kwargs: Forwarded to :func:`compress` (e.g. ``protect_recent``).++ Returns:+ CompressResult with losslessly densified messages.+ """+ kwargs["mode"] = "agent"+ return compress(messages, model=model, model_limit=model_limit, **kwargs)++def _get_pipeline() -> Any:"""Get or create the singleton compression pipeline."""global _pipeline@@ -440,3 +542,164 @@ def _get_pipeline() -> Any:_pipeline = TransformPipeline()logger.debug("Headroom compression pipeline initialized")return _pipeline+++def _get_agent_pipeline() -> Any:+ """Get or create the singleton live-agent pipeline (lossless densify only).++ A ContentRouter with CCR removal and the ML text compressor disabled, so it+ only runs deterministic, query-independent lossless transforms (SmartCrusher+ densification, diff compaction). No CacheAligner stage — agent callers pass+ the full message list each turn and rely on densification determinism for+ prompt-cache stability.+ """+ global _agent_pipeline++ if _agent_pipeline is not None:+ return _agent_pipeline++ with _pipeline_lock:+ if _agent_pipeline is not None:+ return _agent_pipeline++ from headroom.transforms import TransformPipeline+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++ router = ContentRouter(+ ContentRouterConfig(+ ccr_enabled=False, # never remove content the agent will read+ ccr_inject_marker=False, # → emit_opaque_markers off in the engine+ enable_kompress=False, # lossless only: no ML text compression+ smart_crusher_with_compaction=True, # lossless-first densification+ )+ )+ _agent_pipeline = TransformPipeline(transforms=[router])+ logger.debug("Headroom agent (densify-only) pipeline initialized")+ return _agent_pipeline+++def _apply_value_factoring(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:+ """Dictionary-encode low-cardinality columns in every densified string leaf.++ Walks message content and runs :func:`factor_values` on any densified block+ (no-op for non-densified or already-factored strings). Returns a new list;+ inputs are not mutated.+ """+ from .transforms.compaction_codec import factor_values, is_compacted++ def walk(node: Any) -> Any:+ if isinstance(node, str):+ return factor_values(node) if is_compacted(node) else node+ if isinstance(node, list):+ return [walk(x) for x in node]+ if isinstance(node, dict):+ return {k: walk(v) for k, v in node.items()}+ return node++ return [walk(m) for m in messages]+++def _verify_and_revert(+ original: list[dict[str, Any]],+ compressed: list[dict[str, Any]],+) -> tuple[list[dict[str, Any]], int, bool]:+ """Revert any compressed message that is not provably lossless.++ Compares each compressed message to its original. A message is kept only if+ it is unchanged, or every string value it changed is a densified block that+ round-trips back to the original (and carries no removal marker). Anything+ unverifiable is reverted to the original — so the result is never lossy.++ Returns ``(messages, num_reverted, lossless)`` where ``lossless`` is True+ when the final message list contains no surviving removal markers.+ """+ from .transforms.compaction_codec import contains_removal_marker++ if len(original) != len(compressed):+ # Structure changed unexpectedly — fall back to the originals.+ return list(original), len(compressed), True++ out: list[dict[str, Any]] = []+ reverted = 0+ for orig, comp in zip(original, compressed):+ if orig == comp or _message_is_lossless(orig, comp):+ out.append(comp)+ else:+ out.append(orig)+ reverted += 1++ lossless = not any(contains_removal_marker(str(m.get("content", ""))) for m in out)+ return out, reverted, lossless+++def _message_is_lossless(orig: dict[str, Any], comp: dict[str, Any]) -> bool:+ """True if ``comp`` is a lossless densification of ``orig``.++ Walks both messages in parallel collecting (original, compressed) string+ pairs at matching positions; each changed pair must be a densified block+ that expands back to the original JSON. Any structural divergence or+ unverifiable change returns False (caller reverts).+ """+ pairs: list[tuple[str, str]] = []+ if not _collect_string_pairs(orig, comp, pairs):+ return False+ return all(_string_pair_is_lossless(o, c) for o, c in pairs)+++def _collect_string_pairs(orig: Any, comp: Any, pairs: list[tuple[str, str]]) -> bool:+ """Collect aligned string leaves from two parallel structures.++ Returns False if the structures diverge in shape (different types, lengths,+ or dict keys) — a shape we cannot verify and must conservatively revert.+ """+ if isinstance(orig, str) and isinstance(comp, str):+ if orig != comp:+ pairs.append((orig, comp))+ return True+ if isinstance(orig, dict) and isinstance(comp, dict):+ if orig.keys() != comp.keys():+ return False+ return all(_collect_string_pairs(orig[k], comp[k], pairs) for k in orig)+ if isinstance(orig, list) and isinstance(comp, list):+ if len(orig) != len(comp):+ return False+ return all(_collect_string_pairs(o, c, pairs) for o, c in zip(orig, comp))+ # Scalars (int/float/bool/None) must match exactly; transforms only touch+ # strings, so a changed non-string is unexpected.+ return bool(orig == comp)+++def _string_pair_is_lossless(original: str, compressed: str) -> bool:+ """True if ``compressed`` densifies ``original`` reversibly.++ ``original`` is expected to be a JSON array string; ``compressed`` is the+ densified block. Lossless iff it carries no removal marker and expands back+ to the same parsed JSON.+ """+ import json as _json++ from .transforms.compaction_codec import (+ contains_removal_marker,+ expand_compacted,+ is_compacted,+ )++ if contains_removal_marker(compressed):+ return False+ if not is_compacted(compressed):+ return False+ decoded = expand_compacted(compressed)+ if decoded is None:+ return False+ try:+ original_obj = _json.loads(original)+ except (ValueError, TypeError):+ return False+ # Compare CANONICAL JSON, not Python ``==``. ``==`` treats True/1/1.0 as+ # equal, so a bool<->int or int<->float mis-encode would falsely pass this+ # check — the last line of defense before densified output is trusted as+ # lossless. ``sort_keys`` keeps the compare insensitive to object key order+ # while staying type- and value-exact.+ return _json.dumps(original_obj, sort_keys=True, ensure_ascii=False) == _json.dumps(+ decoded, sort_keys=True, ensure_ascii=False+ )diff --git a/headroom/transforms/compaction_codec.py b/headroom/transforms/compaction_codec.pynew file mode 100644index 000000000..64cf42c32--- /dev/null+++ b/headroom/transforms/compaction_codec.py@@ -0,0 +1,415 @@+"""Inverse decoder for SmartCrusher's ``csv-schema`` densification format.++SmartCrusher (the Rust ``compaction`` module) renders a JSON array of objects+into a compact ``[N]{schema}`` header followed by CSV rows. That direction is+*encode-only* in the engine — the original is recoverable only by CCR retrieval+of the stashed blob by hash. For a **live agent** that never wants removal/CCR,+the densified text must instead be programmatically reversible so callers can:++ 1. **Verify losslessness** — round-trip the densified text and compare it to+ the original (the guarantee behind ``compress(mode="agent")``).+ 2. **Re-expand on demand** — a consumer expecting the original ``[{...}]``+ shape can call :func:`expand_compacted` instead of parsing the dense form.++This module is pure-Python (stdlib only) and deliberately mirrors the wire+format emitted by ``crates/headroom-core/.../compaction/formatter.rs`` so the two+stay in lockstep. The cell-encoding contract (post null/empty fix) is:++================== ========================== ============================+Original value Rendered cell Decoded back to+================== ========================== ============================+missing key empty cell key omitted from the dict+JSON ``null`` ``null`` (bare) ``None``+``""`` (empty str) ``""`` (quoted empty) ``""``+literal ``"null"`` ``"null"`` (quoted) ``"null"``+string w/ ``,"``\\n CSV-quoted the string+other string bare the string+number / bool bare ``int``/``float``/``bool``+nested / array CSV-quoted compact JSON the nested object/array+opaque blob ``<<ccr:hash,kind,size>>`` NOT reversible from text+================== ========================== ============================++Schemas declare one column per field as ``name:type`` (or ``name:type?`` when the+column contains nulls/missing). ``name`` may be dotted (``meta.owner``) when the+engine flattened a uniform nested object one level; :func:`expand_compacted`+un-flattens those back into nested dicts.+"""++from __future__ import annotations++import json+from typing import Any++# Bare cells the native formatter emits for an absent key and for JSON null.+# The strings "" and "null" are always CSV-quoted, so neither is ambiguous.+# Kept in one place so the encoder (Rust) and this decoder agree.+MISSING_CELL = ""+NULL_TOKEN = "null"++# Scalar type tags the schema header can declare. ``json`` covers nested+# objects/arrays and mixed-type columns.+_SCALAR_TYPES = frozenset({"int", "float", "bool", "string", "json", "null"})+++def _peel(text: str) -> str:+ """Strip a JSON-string wrapper around a densified block.++ When ContentRouter densifies a tool result whose content was a JSON array,+ it re-serializes the compacted block as a JSON string, so the stored content+ is ``"[N]{...}\\n..."`` (leading quote, escaped newlines) rather than the+ bare block. Peel that wrapper so the same codec handles both forms.+ """+ if text[:1] == '"':+ try:+ inner = json.loads(text)+ except (ValueError, TypeError):+ return text+ if isinstance(inner, str):+ return inner+ return text+++def is_compacted(text: str) -> bool:+ """Return True if ``text`` looks like a ``[N]{schema}`` densified block.++ Structural check (not a heuristic): the first non-empty line must open with+ ``[<digits>]{`` and the brace section must close on that line. Accepts a+ JSON-string-wrapped block. Cheap enough to gate ``expand_compacted``.+ """+ if not text:+ return False+ first = _peel(text).lstrip().split("\n", 1)[0]+ if not first.startswith("["):+ return False+ rbracket = first.find("]")+ if rbracket < 2 or not first[1:rbracket].isdigit():+ return False+ return first[rbracket + 1 : rbracket + 2] == "{" and "}" in first[rbracket:]+++def contains_removal_marker(text: str) -> bool:+ """True if the densified text removed content (CCR marker or dropped rows).++ Either signal means the text alone cannot reconstruct the original — the+ compression was lossy/removal-based, not pure densification.+ """+ return "<<ccr:" in text or "__dropped:" in text+++class _Column:+ __slots__ = ("name", "type", "nullable", "path")++ def __init__(self, decl: str) -> None:+ # Split on the final ':' so dotted names (meta.owner) survive.+ name, _, type_tag = decl.rpartition(":")+ if not name:+ name, type_tag = type_tag, "string"+ self.nullable = type_tag.endswith("?")+ self.type = type_tag[:-1] if self.nullable else type_tag+ if self.type not in _SCALAR_TYPES:+ # Unknown tag → treat as opaque string; keeps the decoder total.+ self.type = "string"+ self.name = name+ self.path = name.split(".")+++def _parse_header(line: str) -> tuple[list[_Column], int, int] | None:+ """Parse the header line.++ Form: ``[N]{c:t,c:t?}`` optionally followed by ``__dict:K`` (K value-factor+ legend lines follow the header) and/or ``__dropped:k``. Returns+ ``(cols, declared_rows, dict_count)``.+ """+ if not line.startswith("["):+ return None+ rb = line.find("]")+ open_brace = line.find("{", rb)+ close_brace = line.find("}", open_brace)+ if rb < 0 or open_brace < 0 or close_brace < open_brace:+ return None+ try:+ declared_rows = int(line[1:rb])+ except ValueError:+ return None+ body = line[open_brace + 1 : close_brace]+ cols = [_Column(d) for d in body.split(",")] if body else []+ dict_count = 0+ for token in line[close_brace + 1 :].split():+ if token.startswith("__dict:"):+ try:+ dict_count = int(token[len("__dict:") :])+ except ValueError:+ dict_count = 0+ return cols, declared_rows, dict_count+++def _iter_rows(blob: str) -> list[list[tuple[bool, str]]]:+ """Split CSV ``blob`` into rows of ``(was_quoted, value)`` cells.++ Hand-rolled rather than the ``csv`` module because we must preserve the+ quoted-vs-bare distinction (``"null"`` is a string; bare ``null`` is JSON+ null) — ``csv.reader`` does not report which cells were quoted.+ Quotes inside quoted fields are escaped by doubling, matching the Rust+ ``csv_quote``; newlines inside quotes are honored.+ """+ rows: list[list[tuple[bool, str]]] = []+ row: list[tuple[bool, str]] = []+ i, n = 0, len(blob)+ while i < n:+ if blob[i] == '"': # quoted cell+ i += 1+ buf: list[str] = []+ while i < n:+ if blob[i] == '"':+ if i + 1 < n and blob[i + 1] == '"':+ buf.append('"')+ i += 2+ continue+ i += 1+ break+ buf.append(blob[i])+ i += 1+ row.append((True, "".join(buf)))+ if i < n and blob[i] == "\r": # tolerate CRLF row endings+ i += 1+ if i < n and blob[i] == ",":+ i += 1+ elif i < n and blob[i] == "\n":+ rows.append(row)+ row = []+ i += 1+ else: # bare cell up to ',' or newline+ j = i+ while j < n and blob[j] not in ",\n":+ j += 1+ cell = blob[i:j]+ # A bare cell never legitimately holds '\r' (the formatter quotes+ # any value containing it), so a trailing '\r' is a CRLF artifact.+ if cell.endswith("\r"):+ cell = cell[:-1]+ row.append((False, cell))+ if j < n and blob[j] == "\n":+ rows.append(row)+ row = []+ i = j + 1+ if row:+ rows.append(row)+ return rows+++def _split_data_rows(blob: str, declared: int) -> list[list[tuple[bool, str]]]:+ """Return the CSV body's data rows, dropping only a trailing-newline artifact.++ The formatter newline-terminates every row, so a single-column row with a+ missing key renders as a bare empty line — a legitimate ``[(False, "")]``+ row, NOT an artifact. Only a lone empty cell that EXCEEDS the declared row+ count is the trailing-newline artifact and is+ dropped; rows with no cells at all are always dropped.+ """+ rows = [r for r in _iter_rows(blob) if r]+ if declared >= 0 and len(rows) == declared + 1 and rows[-1] == [(False, "")]:+ rows.pop()+ return rows+++def _decode_cell(was_quoted: bool, value: str, col: _Column) -> tuple[bool, Any]:+ """Return ``(present, decoded)``. ``present=False`` means the key was absent."""+ if was_quoted:+ if col.type == "json":+ try:+ return True, json.loads(value)+ except (ValueError, TypeError):+ return True, value+ return True, value # quoted "" -> "", quoted "null" -> "null", etc.+ if value == MISSING_CELL:+ return False, None+ if value == NULL_TOKEN:+ return True, None+ if col.type == "int":+ try:+ return True, int(value)+ except ValueError:+ return True, value+ if col.type == "float":+ try:+ return True, float(value)+ except ValueError:+ return True, value+ if col.type == "bool":+ return True, value == "true"+ if col.type == "json":+ try:+ return True, json.loads(value)+ except (ValueError, TypeError):+ return True, value+ return True, value # string / null / unknown+++def _render_raw(was_quoted: bool, value: str) -> str:+ """Reconstruct a cell's exact on-wire text from a parsed ``(quoted, value)``."""+ if was_quoted:+ return '"' + value.replace('"', '""') + '"'+ return value+++def factor_values(text: str) -> str:+ """Dictionary-encode low-cardinality string columns. Lossless and reversible.++ SmartCrusher's CSV elides repeated *keys* (the schema header) but still+ repeats low-cardinality *values* on every row — e.g. a ``search_files``+ result repeats the full file path once per match. This pass replaces such a+ column's cells with integer indices into a ``@col=[...]`` legend, hoisting+ each distinct value out of the row body. A column is encoded only when it+ *strictly* saves bytes (legend + indices < the repeated cells) — no+ cardinality threshold, the byte math decides. :func:`expand_compacted`+ reverses it. Returns ``text`` unchanged when nothing benefits or the input+ is already factored / not densified.++ Preserves a JSON-string wrapper if present (agent tool results are wrapped).+ """+ wrapped = text[:1] == '"'+ inner = _peel(text)+ if not is_compacted(inner) or contains_removal_marker(inner):+ return text+ head, _, rest = inner.partition("\n")+ parsed = _parse_header(head.strip())+ if parsed is None:+ return text+ cols, declared, existing_dict = parsed+ if existing_dict: # already value-factored — idempotent+ return text++ rows = _split_data_rows(rest, declared)+ if not rows:+ return text++ legends: dict[int, list[Any]] = {} # col index -> distinct values (ordered)+ for j, col in enumerate(cols):+ if col.type != "string":+ continue+ order: list[Any] = []+ seen: dict[Any, int] = {}+ raw_bytes = 0+ index_bytes = 0+ for cells in rows:+ if j >= len(cells):+ continue+ was_quoted, value = cells[j]+ if not was_quoted and value in (NULL_TOKEN, MISSING_CELL):+ continue # null / missing stay inline+ # _iter_rows already unescaped quoted cells, so the logical string+ # value is `value` whether or not it was quoted on the wire.+ decoded = value+ raw_bytes += len(_render_raw(was_quoted, value))+ if decoded not in seen:+ seen[decoded] = len(order)+ order.append(decoded)+ index_bytes += len(str(seen[decoded]))+ if not order:+ continue+ legend_json = json.dumps(order, ensure_ascii=False, separators=(",", ":"))+ legend_bytes = len("@") + len(col.name) + len("=") + len(legend_json) + 1+ if legend_bytes + index_bytes < raw_bytes:+ legends[j] = order++ if not legends:+ return text++ # Rebuild: header + __dict:K, then K legend lines, then re-rendered rows.+ close = head.find("}", head.find("{"))+ prefix, trailing = head[: close + 1], head[close + 1 :]+ new_head = f"{prefix} __dict:{len(legends)}{trailing}"++ legend_lines = []+ index_maps: dict[int, dict[Any, int]] = {}+ for j in sorted(legends):+ order = legends[j]+ index_maps[j] = {v: i for i, v in enumerate(order)}+ legend_lines.append(+ f"@{cols[j].name}=" + json.dumps(order, ensure_ascii=False, separators=(",", ":"))+ )++ out_rows = []+ for cells in rows:+ rendered = []+ for j in range(len(cols)):+ if j >= len(cells):+ break+ was_quoted, value = cells[j]+ if j in index_maps and not (not was_quoted and value in (NULL_TOKEN, MISSING_CELL)):+ rendered.append(str(index_maps[j][value]))+ else:+ rendered.append(_render_raw(was_quoted, value))+ out_rows.append(",".join(rendered))++ factored = "\n".join([new_head, *legend_lines, *out_rows]) + "\n"+ return json.dumps(factored, ensure_ascii=False) if wrapped else factored+++def _assign(obj: dict[str, Any], path: list[str], val: Any) -> None:+ """Set ``val`` at a possibly-dotted ``path``, building nested dicts."""+ cur = obj+ for key in path[:-1]:+ nxt = cur.get(key)+ if not isinstance(nxt, dict):+ nxt = {}+ cur[key] = nxt+ cur = nxt+ cur[path[-1]] = val+++def expand_compacted(text: str) -> list[dict[str, Any]] | None:+ """Reconstruct the original array of objects from densified ``text``.++ Returns the list of dicts, or ``None`` if ``text`` is not a parseable+ densified block or carries an irreversible removal marker (CCR / dropped+ rows) — callers treat ``None`` as "cannot losslessly reverse".+ """+ if not is_compacted(text) or contains_removal_marker(text):+ return None+ text = _peel(text)+ head, _, rest = text.partition("\n")+ parsed = _parse_header(head.strip())+ if parsed is None:+ return None+ cols, declared, dict_count = parsed++ # Value-factor legends: ``__dict:K`` declares K ``@col=[...]`` lines that+ # map a low-cardinality column's cells (integer indices) back to values.+ legends: dict[str, list[Any]] = {}+ for _ in range(dict_count):+ legend_line, _, rest = rest.partition("\n")+ name, eq, payload = legend_line.partition("=")+ if not eq or not name.startswith("@"):+ return None+ try:+ legends[name[1:]] = json.loads(payload)+ except (ValueError, TypeError):+ return None++ out: list[dict[str, Any]] = []+ for cells in _split_data_rows(rest, declared):+ if not cells:+ continue+ record: dict[str, Any] = {}+ for idx, col in enumerate(cols):+ if idx >= len(cells):+ break+ was_quoted, value = cells[idx]+ if col.name in legends and not was_quoted:+ # Dict-encoded cell: null -> None, empty -> missing, else index.+ if value == MISSING_CELL:+ continue+ if value == NULL_TOKEN:+ _assign(record, col.path, None)+ continue+ try:+ _assign(record, col.path, legends[col.name][int(value)])+ except (ValueError, IndexError):+ return None+ continue+ present, decoded = _decode_cell(was_quoted, value, col)+ if present:+ _assign(record, col.path, decoded)+ out.append(record)+ return outdiff --git a/tests/test_densify_agent_mode.py b/tests/test_densify_agent_mode.pynew file mode 100644index 000000000..4b7d4b229--- /dev/null+++ b/tests/test_densify_agent_mode.py@@ -0,0 +1,218 @@+"""Live-agent compression mode (``compress(mode='agent')`` / ``densify``).++Agent mode is the densify-only regime for compressing tool results an agent+reads back: no CCR/removal, no ML text compression, a verified losslessness+guarantee, and deterministic (prompt-cache-safe) output. These tests assert+those invariants and that the default ``compress()`` path is unchanged.+"""++from __future__ import annotations++import json++import headroom+from headroom.transforms.compaction_codec import expand_compacted, is_compacted+++def _tool_result_messages(records: list[dict]) -> list[dict]:+ return [+ {"role": "user", "content": "run it"},+ {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": "t1", "name": "q", "input": {}}],+ },+ {+ "role": "user",+ "content": [+ {"type": "tool_result", "tool_use_id": "t1", "content": json.dumps(records)}+ ],+ },+ {"role": "user", "content": "summarize"},+ ]+++def _records(n: int = 80) -> list[dict]:+ return [+ {"id": i, "owner": f"user{i % 5}", "status": "active", "note": f"row padding text {i} here"}+ for i in range(n)+ ]+++def test_densify_compresses_losslessly_without_markers() -> None:+ res = headroom.densify(_tool_result_messages(_records()))+ out = json.dumps(res.messages)+ assert res.tokens_saved > 0+ assert res.lossless is True+ assert res.reverted_messages == 0+ assert "<<ccr:" not in out # no removal markers+ assert "__dropped:" not in out # no dropped rows+++def test_densified_tool_result_round_trips() -> None:+ records = _records()+ res = headroom.densify(_tool_result_messages(records))+ content = res.messages[2]["content"][0]["content"]+ assert is_compacted(content)+ assert expand_compacted(content) == records+++def test_opaque_blob_is_preserved_not_pointerized() -> None:+ blob = "X" * 2000 # well over the 256B opaque threshold+ msgs = [+ {"role": "user", "content": "go"},+ {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": "t2", "name": "read", "input": {}}],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "t2",+ "content": json.dumps({"file": "x", "body": blob}),+ }+ ],+ },+ ]+ res = headroom.densify(msgs)+ out = json.dumps(res.messages)+ assert res.lossless is True+ assert "<<ccr:" not in out+ assert blob in out # the blob the agent needs is still there verbatim+++def test_deterministic_output_for_cache_safety() -> None:+ msgs = _tool_result_messages(_records())+ a = headroom.densify(msgs)+ b = headroom.densify(_tool_result_messages(_records()))+ assert json.dumps(a.messages) == json.dumps(b.messages)+++def test_value_factoring_applied_in_agent_mode() -> None:+ paths = ["./pkg/module_one.py", "./pkg/module_two.py", "./pkg/module_three.py"]+ records = [+ {"path": paths[i % len(paths)], "line": 10 + i, "content": f"call_{i}()"} for i in range(40)+ ]+ res = headroom.densify(_tool_result_messages(records))+ content = res.messages[2]["content"][0]["content"]+ assert "__dict:" in content # legend present+ assert expand_compacted(content) == records # lossless+++def test_default_compress_is_unchanged() -> None:+ res = headroom.compress(_tool_result_messages(_records()), kompress_model="disabled")+ # Default regime does not run lossless verification.+ assert res.lossless is None+ assert res.reverted_messages == 0+ assert res.tokens_saved > 0+++def test_verify_lossless_flag_without_agent_mode() -> None:+ res = headroom.compress(+ _tool_result_messages(_records()), kompress_model="disabled", verify_lossless=True+ )+ assert res.lossless is True+++def test_empty_and_passthrough_inputs() -> None:+ assert headroom.densify([]).messages == []+ short = [{"role": "user", "content": "hi"}]+ assert headroom.densify(short).messages == short+++def test_openai_format_tool_message_is_densified_losslessly() -> None:+ # Generic harness: OpenAI chat format (role='tool', string content) — not+ # Anthropic content blocks. Agent mode must handle it the same way.+ records = [{"id": i, "tag": f"t{i % 4}", "note": f"value padding text {i}"} for i in range(60)]+ msgs = [+ {"role": "user", "content": "do it"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {"id": "c1", "type": "function", "function": {"name": "q", "arguments": "{}"}}+ ],+ },+ {"role": "tool", "tool_call_id": "c1", "content": json.dumps(records)},+ ]+ res = headroom.densify(msgs)+ assert res.lossless is True+ assert res.tokens_saved > 0+ content = res.messages[2]["content"]+ assert is_compacted(content)+ assert expand_compacted(content) == records+++def test_reverse_helpers_are_public() -> None:+ # A consumer of densified output must be able to reverse it from the+ # top-level package, not only a deep submodule path.+ from headroom import expand_compacted as top_expand+ from headroom import is_compacted as top_is++ records = [{"id": i, "tag": f"t{i % 3}"} for i in range(30)]+ res = headroom.densify(_tool_result_messages(records))+ content = res.messages[2]["content"][0]["content"]+ assert top_is(content)+ assert top_expand(content) == records+++def test_verify_rejects_bool_int_conflation() -> None:+ # The losslessness check compares CANONICAL JSON, not Python `==` (which+ # treats True/1/1.0 as equal). A densified block that decodes a bool to an+ # int must NOT be accepted as lossless — else a type mis-encode passes the+ # last line of defense before output is trusted.+ from headroom.compress import _string_pair_is_lossless++ # `[1]{a:int}\n1\n` decodes to [{"a": 1}] — an int, not the original bool.+ assert _string_pair_is_lossless('[{"a":true}]', "[1]{a:int}\n1\n") is False+ # positive control: a faithful int round-trip is still accepted.+ assert _string_pair_is_lossless('[{"a":1}]', "[1]{a:int}\n1\n") is True+++def _count_messages(messages: list[dict]) -> int:+ from headroom.compress import _get_agent_pipeline++ return (+ _get_agent_pipeline()._get_tokenizer("claude-sonnet-4-5-20250929").count_messages(messages)+ )+++def _repeated_path_records() -> list[dict]:+ paths = [f"./src/very/long/package/path/module_{k}/implementation_file.py" for k in range(3)]+ return [{"path": paths[i % 3], "line": i, "text": f"hit {i}"} for i in range(40)]+++def test_tokens_after_counts_value_factored_messages() -> None:+ res = headroom.densify(_tool_result_messages(_repeated_path_records()))+ assert "__dict:" in res.messages[2]["content"][0]["content"]+ assert res.reverted_messages == 0+ assert res.tokens_after == _count_messages(res.messages)+ assert res.tokens_saved == res.tokens_before - res.tokens_after+++def test_tokens_after_counts_reverted_messages(monkeypatch) -> None:+ import importlib++ # `headroom.compress` the attribute is the function; fetch the module.+ compress_module = importlib.import_module("headroom.compress")+ monkeypatch.setattr(compress_module, "_message_is_lossless", lambda orig, comp: False)+ msgs = _tool_result_messages(_repeated_path_records())+ res = headroom.densify(msgs)+ assert res.reverted_messages == 1+ assert res.messages == msgs+ assert res.tokens_after == _count_messages(msgs)+ assert res.tokens_saved == 0+++def test_new_fields_do_not_shift_positional_arguments() -> None:+ # New fields go after `diagnostics` so existing positional construction+ # still binds `diagnostics` where it did before this mode existed.+ from dataclasses import fields++ from headroom.compress import CompressConfig, CompressResult++ config_names = [f.name for f in fields(CompressConfig)]+ assert config_names[-3:] == ["diagnostics", "mode", "verify_lossless"]+ result_names = [f.name for f in fields(CompressResult)]+ assert result_names[-3:] == ["diagnostics", "lossless", "reverted_messages"]diff --git a/tests/test_transforms/test_compaction_codec.py b/tests/test_transforms/test_compaction_codec.pynew file mode 100644index 000000000..67d100f56--- /dev/null+++ b/tests/test_transforms/test_compaction_codec.py@@ -0,0 +1,180 @@+"""Inverse decoder + value-factoring for the ``csv-schema`` densified format.++These exercise the Python codec (``headroom.transforms.compaction_codec``)+against *real* Rust SmartCrusher output so the encoder and decoder stay in+lockstep:++- losslessness of densification is programmatically verifiable (round-trip);+- the null / empty-string / missing-key / literal ``"null"`` distinctions+ survive, matching the native formatter's encoding;+- value-factoring dictionary-encodes low-cardinality columns reversibly and+ only when it saves bytes.+"""++from __future__ import annotations++import json++from headroom.transforms.compaction_codec import (+ contains_removal_marker,+ expand_compacted,+ factor_values,+ is_compacted,+)+from headroom.transforms.smart_crusher import SmartCrusher+++def _crush(records: list[dict]) -> str:+ return SmartCrusher().crush_array_json(json.dumps(records)).get("compacted") or ""+++def test_round_trips_real_rust_output() -> None:+ records = [+ {"id": i, "owner": f"user{i % 5}", "score": i / 3, "active": i % 2 == 0} for i in range(40)+ ]+ comp = _crush(records)+ assert is_compacted(comp)+ assert expand_compacted(comp) == records+++def test_null_empty_missing_and_literal_backslash_n_are_distinct() -> None:+ records: list[dict] = []+ for i in range(40):+ if i == 0:+ records.append({"a": i, "b": i, "c": None}) # null+ elif i == 1:+ records.append({"a": i, "b": i, "c": ""}) # empty string+ elif i == 2:+ records.append({"a": i, "b": i}) # missing key+ elif i == 3:+ records.append({"a": i, "b": i, "c": "\\N"}) # literal backslash-N+ elif i == 4:+ records.append({"a": i, "b": i, "c": "null"}) # literal string "null"+ else:+ records.append({"a": i, "b": i, "c": f"v{i}"})+ comp = _crush(records)+ decoded = expand_compacted(comp)+ assert decoded is not None+ assert "c" not in decoded[2] # missing stays absent+ assert decoded[0]["c"] is None # null+ assert decoded[1]["c"] == "" # empty string+ assert decoded[3]["c"] == "\\N" # literal preserved+ assert decoded[4]["c"] == "null" # quoted literal is a string, not null+ assert decoded == records+++def test_strings_with_commas_quotes_and_unicode() -> None:+ records = [+ {"k": i, "v": v}+ for i, v in enumerate(+ ["a,b", 'he said "hi"', "café 日本語 🔥", "trailing space ", "", "line1"]+ )+ ]+ # pad so the array clears the densification token gate+ records += [{"k": 100 + i, "v": f"pad value {i}"} for i in range(30)]+ comp = _crush(records)+ assert expand_compacted(comp) == records+++def test_nested_object_flattening_round_trips() -> None:+ records = [{"id": i, "meta": {"owner": f"u{i % 3}", "rank": i}} for i in range(30)]+ comp = _crush(records)+ decoded = expand_compacted(comp)+ assert decoded == records+++def test_value_factoring_beats_plain_on_repeated_paths() -> None:+ paths = ["./a/very/long/path/one.py", "./another/long/path/two.py", "./third.py"]+ records = [+ {"path": paths[i % len(paths)], "line": 100 + i, "content": f"def f_{i}(): pass"}+ for i in range(40)+ ]+ plain = _crush(records)+ factored = factor_values(plain)+ assert "__dict:" in factored+ assert len(factored) < len(plain) # strictly smaller+ assert expand_compacted(factored) == records # still lossless+++def test_value_factoring_is_idempotent() -> None:+ records = [{"path": f"./p{i % 3}.py", "line": i, "content": f"x{i}"} for i in range(30)]+ factored = factor_values(_crush(records))+ assert factor_values(factored) == factored+++def test_value_factoring_noop_when_no_benefit() -> None:+ # All-distinct high-entropy column → no repetition to hoist.+ records = [{"id": i, "uuid": f"id-{i}-{'x' * 20}"} for i in range(30)]+ plain = _crush(records)+ assert factor_values(plain) == plain+++def test_value_factoring_preserves_nulls_in_dict_column() -> None:+ records = [+ {"path": (None if i % 7 == 0 else f"./p{i % 3}.py"), "line": i, "content": f"c{i}"}+ for i in range(40)+ ]+ factored = factor_values(_crush(records))+ assert expand_compacted(factored) == records+++def test_removal_marker_makes_text_unreversible() -> None:+ text = "[5]{a:int,blob:string}\n0,<<ccr:abc123,string,1.2KB>>\n"+ assert contains_removal_marker(text)+ assert expand_compacted(text) is None+++def test_json_string_wrapped_block_is_recognized() -> None:+ records = [{"id": i, "tag": f"t{i % 4}"} for i in range(30)]+ bare = _crush(records)+ wrapped = json.dumps(bare) # the form ContentRouter stores for tool results+ assert is_compacted(wrapped)+ assert expand_compacted(wrapped) == records+++def test_non_compacted_text_is_ignored() -> None:+ assert not is_compacted("just some prose, not densified")+ assert expand_compacted("just some prose") is None+ assert factor_values("just some prose") == "just some prose"+++def test_crlf_line_endings_round_trip() -> None:+ # A harness that round-trips the block through a Windows text-mode file (or+ # otherwise normalizes line endings) must still decode. The format is+ # LF-internal; the decoder tolerates CRLF defensively.+ records = [{"id": i, "name": f"n{i % 4}", "v": f"val {i}"} for i in range(30)]+ bare = _crush(records)+ crlf = bare.replace("\n", "\r\n")+ assert expand_compacted(crlf) == records+ assert expand_compacted(crlf) == expand_compacted(bare)+++def test_value_factoring_handles_comma_and_quote_values() -> None:+ # Repeated values that need CSV-quoting must still factor losslessly.+ vals = ["a, b.py", 'has "quotes"', "c/d.py"]+ records = [{"path": vals[i % len(vals)], "line": i, "content": f"x{i}"} for i in range(30)]+ factored = factor_values(_crush(records))+ assert "__dict:" in factored+ assert expand_compacted(factored) == records+++def test_single_column_missing_row_is_not_dropped() -> None:+ # Native encoding: missing key is a bare empty cell, null is bare `null`,+ # and "" / "null" are quoted. A trailing single-column missing row renders+ # as a bare empty line; the declared row count keeps it from being dropped+ # as a trailing-newline artifact.+ block = '[4]{k:string?}\n""\nnull\n"null"\n\n'+ assert expand_compacted(block) == [{"k": ""}, {"k": None}, {"k": "null"}, {}]+++def test_trailing_newline_beyond_declared_count_is_trimmed() -> None:+ # A blank line BEYOND the declared row count is the trailing-newline+ # artifact and must be dropped (not decoded as a spurious extra null row).+ assert expand_compacted("[2]{k:string?}\na\nb\n\n") == [{"k": "a"}, {"k": "b"}]+++def test_single_column_null_survives_value_factoring() -> None:+ block = "[3]{k:string?}\nx\ny\nnull\n"+ factored = factor_values(block)+ assert expand_compacted(factored) == expand_compacted(block)+ assert expand_compacted(factored) == [{"k": "x"}, {"k": "y"}, {"k": None}]