Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(proxy): add --lossless no-CCR mode with format-native compaction (#1721)
Tejas Choprac75ebdee6df9
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex 1d479d106..6377407bb 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -297,6 +297,16 @@ def dashboard(port: int, no_open: bool) -> None:envvar="HEADROOM_NO_CCR_MARKER",help=("Don't add CCR retrieval markers to compressed content. Env: HEADROOM_NO_CCR_MARKER."),)+ "--lossless",+ is_flag=True,+ envvar="HEADROOM_LOSSLESS",+ help=(+ "No-CCR lossless mode: compress tool outputs with format-native lossless "+ "compaction (and marker-free SmartCrusher) without emitting any CCR "+ "retrieval marker, so no MCP retrieve tool is needed. Env: HEADROOM_LOSSLESS=1."+ ),+)@click.option("--no-ccr-proactive-expansion",is_flag=True,@@ -844,6 +854,7 @@ def proxy(tpm: int | None,no_ccr_inject_tool: bool,no_ccr_marker: bool,+ lossless: bool,no_ccr_proactive_expansion: bool,proxy_extension: tuple[str, ...],no_subscription_tracking: bool,@@ -1086,6 +1097,7 @@ def proxy(# CCR fully on; each flag flips one dataclass default to False.ccr_inject_tool=not no_ccr_inject_tool,ccr_inject_marker=not no_ccr_marker,+ lossless=lossless,ccr_proactive_expansion=not no_ccr_proactive_expansion,# Flatten repeat-flag tuple AND any comma-separated values inside it.# `--proxy-extension a,b --proxy-extension c` and `HEADROOM_PROXY_EXTENSIONS=a,b,c`diff --git a/headroom/proxy/models.py b/headroom/proxy/models.pyindex 34b6e8e25..b1b9fd0a8 100644--- a/headroom/proxy/models.py+++ b/headroom/proxy/models.py@@ -180,6 +180,8 @@ class ProxyConfig:# CLI: --force-kompress-all; env: HEADROOM_FORCE_KOMPRESS_ALL=1.force_kompress_all: bool = False+ lossless: bool = False # CLI: --lossless; env: HEADROOM_LOSSLESS=1. No-CCR mode: compress without any retrieval marker.+# Code graph live watcher (triggers incremental reindex on file changes)code_graph_watcher: bool = Falsediff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 5d861e6d3..bab13668d 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -674,7 +674,18 @@ class HeadroomProxy(),ccr_inject_marker=config.ccr_inject_marker,force_kompress_all=config.force_kompress_all,+ lossless=config.lossless,)+ # No-CCR lossless mode: compress tool outputs with format-native+ # lossless compaction and marker-free SmartCrusher, and suppress every+ # retrieval marker + the retrieve-tool injection so no MCP round-trip is+ # needed. Mirrors the force_kompress_all wiring precedent.+ if config.lossless:+ router_config.lossless = True+ router_config.smart_crusher_lossless_only = True+ router_config.ccr_inject_marker = False+ if hasattr(config, "ccr_inject_tool"):+ config.ccr_inject_tool = Falseif config.disable_kompress:router_config.enable_kompress = False# Opt-in restore of the legacy behaviour: send fall-through content@@ -4080,6 +4091,7 @@ def _proxy_config_from_env() -> ProxyConfig:disable_kompress_anthropic=_get_env_optional_bool("HEADROOM_DISABLE_KOMPRESS_ANTHROPIC"),disable_kompress_openai=_get_env_optional_bool("HEADROOM_DISABLE_KOMPRESS_OPENAI"),force_kompress_all=_get_env_bool("HEADROOM_FORCE_KOMPRESS_ALL", False),+ lossless=_get_env_bool("HEADROOM_LOSSLESS", False),max_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", 500),max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", 100),keepalive_expiry=_get_env_float("HEADROOM_KEEPALIVE_EXPIRY", 90.0),@@ -4574,6 +4586,16 @@ if __name__ == "__main__":"Also settable via HEADROOM_FORCE_KOMPRESS_ALL=1."),)+ parser.add_argument(+ "--lossless",+ action="store_true",+ help=(+ "No-CCR lossless mode: compress LOG/SEARCH/DIFF tool outputs with "+ "format-native lossless compaction (and marker-free SmartCrusher) "+ "without emitting any CCR retrieval marker, so no MCP retrieve tool "+ "is needed. Also settable via HEADROOM_LOSSLESS=1."+ ),+ )parser.add_argument("--exclude-tools",default=None,@@ -4655,6 +4677,7 @@ if __name__ == "__main__":force_kompress_all = args.force_kompress_all or _get_env_bool("HEADROOM_FORCE_KOMPRESS_ALL", False)+ lossless = getattr(args, "lossless", False) or _get_env_bool("HEADROOM_LOSSLESS", False)# Set OpenRouter API key from CLI if providedif hasattr(args, "openrouter_api_key") and args.openrouter_api_key:@@ -4710,6 +4733,7 @@ if __name__ == "__main__":disable_kompress_anthropic=disable_kompress_anthropic,disable_kompress_openai=disable_kompress_openai,force_kompress_all=force_kompress_all,+ lossless=lossless,# Connection pool settingsmax_connections=_get_env_int("HEADROOM_MAX_CONNECTIONS", args.max_connections),max_keepalive_connections=_get_env_int("HEADROOM_MAX_KEEPALIVE", args.max_keepalive),diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 191b1ec1b..fa13c044d 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -762,6 +762,13 @@ class ContentRouterConfig:# Route ALL compressible content to Kompress, skipping per-type selection.# Tool exclusion (Read/Glob/...) and reversibility gates still apply.force_kompress_all: bool = False++ # No-CCR lossless mode. When True the router compresses LOG/SEARCH/DIFF+ # content with format-native lossless compaction (headroom.transforms.+ # lossless_compaction) instead of the lossy Rust drop path, and never+ # emits a `<<ccr:…>>` / `Retrieve …` retrieval marker. SmartCrusher is+ # additionally forced marker-free via smart_crusher_lossless_only.+ lossless: bool = Falsemixed_content_threshold: int = 2 # Min types to consider mixedmin_section_tokens: int = 20 # Min tokens to compress a section@@ -1115,6 +1122,13 @@ class ContentRouter(Transform):rule in the audit doc."""self.config = config or ContentRouterConfig()+ # No-CCR lossless mode is self-consistent regardless of how the config+ # was built: force marker-free output and marker-free SmartCrusher so+ # the invariant (no `<<ccr:…>>` / `Retrieve …`) holds even when a caller+ # constructs ContentRouterConfig(lossless=True) directly.+ if self.config.lossless:+ self.config.ccr_inject_marker = False+ self.config.smart_crusher_lossless_only = Trueself._observer = observer# Lazy-loaded compressors@@ -1616,6 +1630,31 @@ class ContentRouter(Transform):strategy_chain: list[str] = [strategy.value]error: str | None = None+ # No-CCR lossless mode: LOG/SEARCH/DIFF get format-native lossless+ # compaction instead of the lossy Rust drop path, so the output stays+ # marker-free (no `<<ccr:…>>` / `Retrieve …`) and fully recoverable.+ # SMART_CRUSHER relies on smart_crusher_lossless_only (wired elsewhere);+ # KOMPRESS/TEXT/CODE_AWARE/PASSTHROUGH pass through unchanged here in+ # Stage A. The reversibility + size gate lives in compact_lossless,+ # which returns the original when it can't safely shrink it.+ if self.config.lossless and strategy in (+ CompressionStrategy.LOG,+ CompressionStrategy.SEARCH,+ CompressionStrategy.DIFF,+ ):+ from headroom.transforms.lossless_compaction import compact_lossless++ kind = {+ CompressionStrategy.LOG: "log",+ CompressionStrategy.SEARCH: "search",+ CompressionStrategy.DIFF: "diff",+ }[strategy]+ try:+ compacted = compact_lossless(content, kind)+ except Exception:+ compacted = content+ return compacted, len(compacted.split()), [f"lossless_{kind}"]+try:if strategy == CompressionStrategy.CODE_AWARE:if self.config.enable_code_aware:@@ -2276,7 +2315,11 @@ class ContentRouter(Transform):)if is_kompress_available():- return KompressCompressor(config=KompressConfig(model_id=model_id))+ return KompressCompressor(+ config=KompressConfig(+ model_id=model_id, enable_ccr=self.config.ccr_inject_marker+ )+ )except ImportError:passreturn None@@ -2284,10 +2327,21 @@ class ContentRouter(Transform):# Default path — exactly as before, cached on selfif self._kompress is None:try:- from .kompress_compressor import KompressCompressor, is_kompress_available+ from .kompress_compressor import (+ KompressCompressor,+ KompressConfig,+ is_kompress_available,+ )if is_kompress_available():- self._kompress = KompressCompressor()+ # Honor the router's marker policy. In no-CCR / lossless mode+ # (ccr_inject_marker=False) Kompress still compresses (lossy),+ # but must NOT append a `Retrieve more: hash=` marker or write+ # to the CCR store — otherwise the no-MCP guarantee breaks.+ # Matches how search/log/diff/code receive enable_ccr.+ self._kompress = KompressCompressor(+ config=KompressConfig(enable_ccr=self.config.ccr_inject_marker)+ )except ImportError:logger.debug("Kompress dependencies not available")return self._kompressdiff --git a/headroom/transforms/lossless_compaction.py b/headroom/transforms/lossless_compaction.pynew file mode 100644index 000000000..4cdcfac98--- /dev/null+++ b/headroom/transforms/lossless_compaction.py@@ -0,0 +1,251 @@+"""Format-native, reversible lossless compaction for no-CCR proxy mode.++Every helper here is pure stdlib and keeps its output *looking like its own+type* — grep stays grep, logs stay logs, diffs stay diffs. No retrieval+marker (``<<ccr:…>>`` / ``Retrieve …``) is ever emitted, so the proxy needs+no MCP retrieve round-trip to stay recoverable.++The reversible transforms ship with exact inverses and are self-checked at+runtime by :func:`compact_lossless`: if a round-trip does not reproduce the+original (modulo intentionally-dropped non-semantic bits such as ANSI color)+or the result is not actually smaller, the original content is returned+unchanged. Nothing here raises.+"""++from __future__ import annotations++import re++__all__ = [+ "strip_ansi",+ "collapse_runs",+ "expand_runs",+ "is_run_collapsed",+ "search_heading",+ "search_unheading",+ "diff_strip_index",+ "compact_lossless",+]++# ANSI CSI SGR (color/style) escape sequences: ESC [ ... m. Color is+# non-semantic, so stripping it is a safe (one-way) lossless-of-meaning op.+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")++# syslog-style run-collapse marker. The count is captured for exact inversion.+_RUN_MARKER_RE = re.compile(r"^\.\.\. \(repeated (\d+) times\)$")++# grep/ripgrep default row shape: ``path:line:content``. ``line`` is digits;+# ``path`` must not itself look like ``line:content`` (i.e. not start with a+# bare number) so we don't mis-split a heading-form ``line:content`` row.+_GREP_ROW_RE = re.compile(r"^(?P<path>[^\n:]+):(?P<line>\d+):(?P<content>.*)$")+# heading-form data row (``line:content``) produced by search_heading.+_HEADING_ROW_RE = re.compile(r"^(?P<line>\d+):(?P<content>.*)$")++# unified-diff ``index <sha>..<sha> <mode>`` line. The diff still applies+# without it (git only uses it for rename/blob bookkeeping).+_DIFF_INDEX_RE = re.compile(r"^index [0-9a-fA-F]+\.\.[0-9a-fA-F]+( [0-7]+)?$")+++def strip_ansi(text: str) -> str:+ """Remove ANSI CSI/SGR (color) escape sequences. Color is non-semantic."""+ return _ANSI_RE.sub("", text)+++def _split_keep_trailing(text: str) -> tuple[list[str], bool]:+ """Split into lines, remembering whether a trailing newline was present.++ Returns (lines, had_trailing_newline). This lets the run helpers rejoin+ byte-exactly instead of always appending or always dropping a newline.+ """+ if text == "":+ return [], False+ had_trailing = text.endswith("\n")+ body = text[:-1] if had_trailing else text+ return body.split("\n"), had_trailing+++def _join(lines: list[str], had_trailing: bool) -> str:+ out = "\n".join(lines)+ if had_trailing:+ out += "\n"+ return out+++def collapse_runs(text: str) -> str:+ """Collapse runs of >=2 identical consecutive lines (syslog convention).++ A run of N (N>=2) identical lines becomes the line once followed by+ ``... (repeated N times)``. Exact inverse: :func:`expand_runs`.+ """+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ i = 0+ n = len(lines)+ while i < n:+ j = i+ while j + 1 < n and lines[j + 1] == lines[i]:+ j += 1+ run_len = j - i + 1+ if run_len >= 2:+ out.append(lines[i])+ out.append(f"... (repeated {run_len} times)")+ else:+ out.append(lines[i])+ i = j + 1+ return _join(out, had_trailing)+++def expand_runs(text: str) -> str:+ """Exact inverse of :func:`collapse_runs`."""+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ i = 0+ n = len(lines)+ while i < n:+ line = lines[i]+ if i + 1 < n:+ m = _RUN_MARKER_RE.match(lines[i + 1])+ if m:+ count = int(m.group(1))+ out.extend([line] * count)+ i += 2+ continue+ out.append(line)+ i += 1+ return _join(out, had_trailing)+++def is_run_collapsed(text: str) -> bool:+ """True if any run-collapse marker line is present."""+ for line in text.split("\n"):+ if _RUN_MARKER_RE.match(line):+ return True+ return False+++def search_heading(text: str) -> str:+ """Convert grep ``path:line:content`` rows into ripgrep --heading form.++ Consecutive rows sharing a path collapse to the path once on its own line+ (a *header* line), then ``line:content`` rows beneath it. Lines that don't+ match the ``path:line:content`` shape are passed through untouched. No+ blank separators are inserted (they would be ambiguous with passthrough+ content), keeping the transform exactly reversible via+ :func:`search_unheading`.+ """+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ current_path: str | None = None+ for line in lines:+ m = _GREP_ROW_RE.match(line)+ if m:+ path = m.group("path")+ if path != current_path:+ out.append(path)+ current_path = path+ out.append(f"{m.group('line')}:{m.group('content')}")+ else:+ # Any non-grep-row line ends the current file grouping.+ out.append(line)+ current_path = None+ return _join(out, had_trailing)+++def search_unheading(text: str) -> str:+ """Exact inverse of :func:`search_heading`.++ A *header* line is any line that is not itself a ``line:content`` data row+ and is immediately followed by at least one ``line:content`` data row; it+ is consumed (not re-emitted) and its text becomes the ``path`` prefix for+ the data rows that follow, until a non-data line appears.+ """+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out: list[str] = []+ current_path: str | None = None+ n = len(lines)+ i = 0+ while i < n:+ line = lines[i]+ data = _HEADING_ROW_RE.match(line)+ if current_path is not None and data:+ out.append(f"{current_path}:{data.group('line')}:{data.group('content')}")+ i += 1+ continue+ # Not a data row under an active header. Decide if THIS line is a new+ # header: it must not be a data row itself and must be followed by a+ # data row. If so, consume it as the path prefix (do not emit).+ if not data and i + 1 < n and _HEADING_ROW_RE.match(lines[i + 1]):+ current_path = line+ i += 1+ continue+ # Plain passthrough line (or a stray data row with no header): emit it+ # verbatim and clear any active grouping.+ current_path = None+ out.append(line)+ i += 1+ return _join(out, had_trailing)+++def diff_strip_index(text: str) -> str:+ """Drop ``index <sha>..<sha>`` lines from a unified diff (still applies)."""+ lines, had_trailing = _split_keep_trailing(text)+ if not lines:+ return text+ out = [line for line in lines if not _DIFF_INDEX_RE.match(line)]+ return _join(out, had_trailing)+++def _smaller(candidate: str, original: str) -> bool:+ return len(candidate) < len(original)+++def compact_lossless(content: str, kind: str) -> str:+ """Dispatch format-native lossless compaction by ``kind``.++ ``kind`` in {'log', 'search', 'diff', 'text'}. For reversible kinds the+ round-trip is verified internally (modulo the intentionally-dropped+ non-semantic bits, e.g. ANSI color for logs); if verification fails or the+ result is not smaller, the original content is returned unchanged. Never+ raises; unknown kinds pass through.+ """+ if not content:+ return content+ try:+ if kind == "log":+ # ANSI is non-semantic and dropped one-way; run-collapse must be+ # exactly reversible against the de-ANSI'd baseline.+ baseline = strip_ansi(content)+ candidate = collapse_runs(baseline)+ if expand_runs(candidate) != baseline:+ return content+ return candidate if _smaller(candidate, content) else content++ if kind == "search":+ candidate = search_heading(content)+ if search_unheading(candidate) != content:+ return content+ return candidate if _smaller(candidate, content) else content++ if kind == "diff":+ # Purely subtractive of non-semantic bookkeeping lines; the+ # remaining hunks still apply. No exact inverse needed.+ candidate = diff_strip_index(content)+ return candidate if _smaller(candidate, content) else content++ if kind == "text":+ # Collapse blank-line runs; reversible against itself.+ candidate = collapse_runs(content)+ if expand_runs(candidate) != content:+ return content+ return candidate if _smaller(candidate, content) else content+ except Exception:+ return content+ return contentdiff --git a/tests/test_lossless_mode.py b/tests/test_lossless_mode.pynew file mode 100644index 000000000..65bddd325--- /dev/null+++ b/tests/test_lossless_mode.py@@ -0,0 +1,345 @@+"""Tests for no-CCR --lossless proxy mode (Stage A).++Covers:+ * flag plumbing (ProxyConfig field, CLI option parses),+ * reversibility of each format-native lossless compaction,+ * end-to-end ContentRouter(lossless=True) invariant: smaller output, NO+ ``<<ccr:`` / ``Retrieve `` marker, and full recoverability.+"""++from __future__ import annotations++from headroom.proxy.models import ProxyConfig+from headroom.transforms.content_router import (+ CompressionStrategy,+ ContentRouter,+ ContentRouterConfig,+)+from headroom.transforms.lossless_compaction import (+ collapse_runs,+ compact_lossless,+ diff_strip_index,+ expand_runs,+ is_run_collapsed,+ search_heading,+ search_unheading,+ strip_ansi,+)+++# --------------------------------------------------------------------------+# Flag plumbing+# --------------------------------------------------------------------------+def test_proxyconfig_has_lossless_field() -> None:+ assert ProxyConfig().lossless is False+ assert ProxyConfig(lossless=True).lossless is True+++def test_content_router_config_has_lossless_field() -> None:+ assert ContentRouterConfig().lossless is False+ cfg = ContentRouterConfig(lossless=True)+ assert cfg.lossless is True+++def test_router_lossless_forces_marker_free_config() -> None:+ # Building the router normalizes the config so the no-CCR invariant holds+ # regardless of how ContentRouterConfig was constructed.+ router = ContentRouter(ContentRouterConfig(lossless=True))+ assert router.config.ccr_inject_marker is False+ assert router.config.smart_crusher_lossless_only is True+++def test_cli_proxy_option_parses_lossless() -> None:+ from click.testing import CliRunner++ from headroom.cli.proxy import proxy++ runner = CliRunner()+ result = runner.invoke(proxy, ["--help"])+ assert result.exit_code == 0+ assert "--lossless" in result.output+++# --------------------------------------------------------------------------+# Reversibility: collapse_runs / expand_runs+# --------------------------------------------------------------------------+def test_collapse_expand_runs_byte_roundtrip() -> None:+ log = (+ "starting worker\n"+ "connection refused\n"+ "connection refused\n"+ "connection refused\n"+ "connection refused\n"+ "connection refused\n"+ "retrying\n"+ "retrying\n"+ "done\n"+ )+ collapsed = collapse_runs(log)+ assert is_run_collapsed(collapsed)+ assert len(collapsed) < len(log)+ assert expand_runs(collapsed) == log+++def test_collapse_runs_no_trailing_newline_roundtrip() -> None:+ log = "a\na\na\nb"+ assert expand_runs(collapse_runs(log)) == log+++def test_collapse_runs_singletons_untouched() -> None:+ log = "one\ntwo\nthree\n"+ assert collapse_runs(log) == log+ assert not is_run_collapsed(log)+++def test_collapse_runs_empty() -> None:+ assert collapse_runs("") == ""+ assert expand_runs("") == ""+++# --------------------------------------------------------------------------+# Reversibility: search_heading / search_unheading+# --------------------------------------------------------------------------+def test_search_heading_unheading_roundtrip() -> None:+ grep = (+ "src/app.py:10:def main():\n"+ "src/app.py:11: run()\n"+ "src/app.py:42: return 0\n"+ "src/util.py:3:import os\n"+ "src/util.py:9:import sys\n"+ )+ headed = search_heading(grep)+ # heading form: each path appears once as its own header line+ assert headed.count("src/app.py") == 1+ assert headed.count("src/util.py") == 1+ assert search_unheading(headed) == grep+++def test_search_heading_smaller_for_repeated_paths() -> None:+ grep = "\n".join(f"a/very/long/path/module.py:{i}:line{i}" for i in range(1, 30)) + "\n"+ headed = search_heading(grep)+ assert len(headed) < len(grep)+ assert search_unheading(headed) == grep+++def test_search_heading_leaves_non_matching_lines() -> None:+ text = "just some prose\nnot a grep row at all\n"+ assert search_heading(text) == text+ assert search_unheading(text) == text+++def test_search_heading_mixed_content_roundtrip() -> None:+ grep = "banner line\nsrc/a.py:1:x\nsrc/a.py:2:y\nmiddle prose\nsrc/b.py:5:z\n"+ headed = search_heading(grep)+ assert search_unheading(headed) == grep+++# --------------------------------------------------------------------------+# strip_ansi+# --------------------------------------------------------------------------+def test_strip_ansi_removes_only_escapes() -> None:+ colored = "\x1b[31mERROR\x1b[0m: boom \x1b[1mbold\x1b[0m end"+ assert strip_ansi(colored) == "ERROR: boom bold end"+ plain = "no escapes here : 1:2:3"+ assert strip_ansi(plain) == plain+++# --------------------------------------------------------------------------+# diff_strip_index+# --------------------------------------------------------------------------+def test_diff_strip_index_keeps_plus_minus() -> None:+ diff = (+ "diff --git a/f.py b/f.py\n"+ "index 0123abc..def4567 100644\n"+ "--- a/f.py\n"+ "+++ b/f.py\n"+ "@@ -1,3 +1,3 @@\n"+ " context\n"+ "-old line\n"+ "+new line\n"+ )+ stripped = diff_strip_index(diff)+ assert "index 0123abc..def4567" not in stripped+ assert "-old line" in stripped+ assert "+new line" in stripped+ assert "@@ -1,3 +1,3 @@" in stripped+ assert "--- a/f.py" in stripped+ assert len(stripped) < len(diff)+++# --------------------------------------------------------------------------+# compact_lossless dispatch + safety gates+# --------------------------------------------------------------------------+def test_compact_lossless_log_roundtrips_modulo_ansi() -> None:+ log = "\x1b[31mfail\x1b[0m\n" + "fail\n" * 5+ out = compact_lossless(log, "log")+ # recoverable modulo ANSI: expand back == de-ANSI'd original+ assert expand_runs(out) == strip_ansi(log)+ assert len(out) < len(log)+++def test_compact_lossless_returns_original_when_not_smaller() -> None:+ # No repeats, no ANSI -> nothing to gain; returns original unchanged.+ log = "line one\nline two\nline three\n"+ assert compact_lossless(log, "log") == log+++def test_compact_lossless_search() -> None:+ grep = "\n".join(f"pkg/mod.py:{i}:code{i}" for i in range(1, 20)) + "\n"+ out = compact_lossless(grep, "search")+ assert len(out) < len(grep)+ assert search_unheading(out) == grep+++def test_compact_lossless_unknown_kind_passthrough() -> None:+ assert compact_lossless("whatever", "mystery") == "whatever"+++def test_compact_lossless_never_raises_on_empty() -> None:+ for kind in ("log", "search", "diff", "text"):+ assert compact_lossless("", kind) == ""+++# --------------------------------------------------------------------------+# End-to-end: ContentRouter(lossless=True) invariant+# --------------------------------------------------------------------------+def _assert_no_marker(text: str) -> None:+ assert "<<ccr:" not in text+ assert "Retrieve " not in text+++def test_router_lossless_log_strategy_no_marker_and_recoverable() -> None:+ # Drive the LOG strategy directly: detection is content-dependent, but the+ # router's lossless disposition for an explicit LOG strategy must apply+ # format-native lossless compaction with no marker and full recovery.+ router = ContentRouter(ContentRouterConfig(lossless=True))+ log = (+ "[info] boot sequence start\n"+ + "connection refused by upstream service alpha\n" * 40+ + "[info] boot sequence complete\n"+ )+ out, _tokens, chain = router._apply_strategy_to_content(+ log, CompressionStrategy.LOG, context=""+ )+ _assert_no_marker(out)+ assert len(out) < len(log)+ assert chain == ["lossless_log"]+ # every original line recoverable (no ANSI here, so exact)+ assert expand_runs(out) == log+++def test_router_lossless_diff_strategy_no_marker() -> None:+ router = ContentRouter(ContentRouterConfig(lossless=True))+ diff = (+ "diff --git a/f.py b/f.py\n"+ "index 0123abc..def4567 100644\n"+ "--- a/f.py\n"+ "+++ b/f.py\n"+ "@@ -1,2 +1,2 @@\n"+ "-old\n"+ "+new\n"+ )+ out, _tokens, chain = router._apply_strategy_to_content(+ diff, CompressionStrategy.DIFF, context=""+ )+ _assert_no_marker(out)+ assert chain == ["lossless_diff"]+ assert "index 0123abc..def4567" not in out+ assert "-old" in out and "+new" in out+++def test_router_lossless_search_no_marker_and_recoverable() -> None:+ router = ContentRouter(ContentRouterConfig(lossless=True))+ grep = (+ "\n".join(+ f"headroom/transforms/content_router.py:{i}: identifier_{i} = compute()"+ for i in range(1, 60)+ )+ + "\n"+ )+ result = router.compress(grep, context="")+ out = result.compressed+ _assert_no_marker(out)+ # identifiers still present / recoverable+ if result.strategy_used == CompressionStrategy.SEARCH:+ assert search_unheading(out) == grep+ # at minimum, no data lost and no marker+ for i in (1, 30, 59):+ assert f"identifier_{i}" in out+++def test_router_lossless_never_emits_marker_various_inputs() -> None:+ router = ContentRouter(ContentRouterConfig(lossless=True))+ samples = [+ "err\n" * 100,+ "\n".join(f"a/b/c.py:{i}:x{i}" for i in range(50)),+ "plain prose line\n" * 30,+ ]+ for s in samples:+ _assert_no_marker(router.compress(s, context="").compressed)+++# --------------------------------------------------------------------------+# Token-delta measurement (informational)+# --------------------------------------------------------------------------+def test_measure_token_deltas(capsys) -> None: # type: ignore[no-untyped-def]+ log = (+ "[warn] disk usage high\n"+ + "\x1b[33mretrying connection to db-primary\x1b[0m\n" * 50+ + "[info] recovered\n"+ )+ grep = (+ "\n".join(+ f"headroom/proxy/server.py:{i}: router_config.value_{i} = {i}" for i in range(1, 80)+ )+ + "\n"+ )+ log_out = compact_lossless(log, "log")+ grep_out = compact_lossless(grep, "search")+ # chars are what the byte-size gate optimizes and what subword tokenizers+ # track closely; whitespace word-count is reported for direction only.+ log_c0, log_c1 = len(log), len(log_out)+ grep_c0, grep_c1 = len(grep), len(grep_out)+ log_t0, log_t1 = len(log.split()), len(log_out.split())+ grep_t0, grep_t1 = len(grep.split()), len(grep_out.split())+ print(+ f"\n[lossless deltas] "+ f"log: {log_c0}->{log_c1} chars "+ f"({100 * (log_c0 - log_c1) / log_c0:.1f}% saved), {log_t0}->{log_t1} words; "+ f"grep: {grep_c0}->{grep_c1} chars "+ f"({100 * (grep_c0 - grep_c1) / grep_c0:.1f}% saved), {grep_t0}->{grep_t1} words"+ )+ assert log_c1 < log_c0+ assert grep_c1 < grep_c0+++def test_lossless_mode_builds_kompress_marker_free(monkeypatch) -> None:+ """In lossless (no-CCR) mode Kompress must be built with enable_ccr=False so+ it never appends a `Retrieve more: hash=` marker or writes the CCR store —+ the agent has no MCP tool to redeem it. Kompress still runs (lossy); only+ the marker/store is suppressed. Verified WITHOUT loading the model.+ """+ import headroom.transforms.kompress_compressor as kc+ from headroom.transforms.content_router import ContentRouter, ContentRouterConfig++ captured: dict[str, object] = {}++ class _FakeKompress:+ def __init__(self, config=None) -> None:+ captured["enable_ccr"] = getattr(config, "enable_ccr", None)++ def is_ready(self) -> bool:+ return False++ def ensure_background_load(self) -> None:+ pass++ monkeypatch.setattr(kc, "is_kompress_available", lambda: True)+ monkeypatch.setattr(kc, "KompressCompressor", _FakeKompress)++ ContentRouter(ContentRouterConfig(lossless=True))._get_kompress()+ assert captured["enable_ccr"] is False # marker suppressed in lossless mode++ captured.clear()+ ContentRouter(ContentRouterConfig(lossless=False))._get_kompress()+ assert captured["enable_ccr"] is True # normal mode: unchanged (marker on)