Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): bound the router's compression cache, keep the rest in SQLite (#4357)
Tejas Choprab196424e35df
diff --git a/headroom/cache/result_store.py b/headroom/cache/result_store.pynew file mode 100644index 000000000..0ec978eb8--- /dev/null+++ b/headroom/cache/result_store.py@@ -0,0 +1,159 @@+"""SQLite tier behind the content router's in-memory compression cache.++The router keeps recent results in a byte-budgeted LRU; this file holds the rest,+so an entry the budget evicted comes back from disk instead of being compressed+again (which costs a Kompress pass and can give different bytes, rewriting a+cached prompt prefix). It also survives a restart and is shared by worker+processes (WAL), like the CCR store; WAL needs a local filesystem.++Each router reads and writes under its own namespace (a digest of its config and+the headroom version), so routers configured differently, or a restart with+other settings, never serve each other's output. Results carrying a CCR marker+are not written here: their originals live in the CCR store, on its own clock.++The file holds compressed tool output, so it follows the CCR store's rules:+owner-only, ``workspace_dir()/compression_cache.db`` (``HEADROOM_COMPRESSION_CACHE_PATH``+to move it), expired rows deleted on open, never opened in stateless mode, and+``HEADROOM_COMPRESSION_CACHE_DISK=0`` keeps the cache in memory only. A database+error is a miss, never a failed request.+"""++from __future__ import annotations++import logging+import os+import sqlite3+import threading+import time+from pathlib import Path++from ..fileperms import connect_private_sqlite++logger = logging.getLogger(__name__)++_SCHEMA = """+CREATE TABLE IF NOT EXISTS results (+ k TEXT PRIMARY KEY,+ text TEXT NOT NULL,+ ratio REAL NOT NULL,+ strategy TEXT NOT NULL,+ expires REAL NOT NULL+);+CREATE INDEX IF NOT EXISTS idx_results_expires ON results (expires);+"""+#: Expired rows are deleted every this many writes, and on open.+_PURGE_EVERY = 1000+++class ResultStore:+ """Compressed results by key; an entry expires ``ttl_seconds`` after it was+ last read or written. Thread-safe; several processes may share the file."""++ def __init__(self, path: Path, ttl_seconds: float) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ self._path = path+ self._ttl = ttl_seconds+ self._lock = threading.Lock()+ self._writes = 0+ self._warned = False+ self._db = connect_private_sqlite(+ path, what="compression cache", check_same_thread=False, isolation_level=None+ )+ self._db.execute("PRAGMA busy_timeout=5000")+ self._db.execute("PRAGMA journal_mode=WAL")+ self._db.execute("PRAGMA synchronous=NORMAL")+ self._db.executescript(_SCHEMA)+ # Sessions that ended while the proxy was down go now.+ self._db.execute("DELETE FROM results WHERE expires <= ?", (time.time(),))++ def get(self, key: str) -> tuple[str, float, str] | None:+ now = time.time()+ try:+ with self._lock:+ row = self._db.execute(+ "SELECT text, ratio, strategy, expires FROM results "+ "WHERE k = ? AND expires > ?",+ (key, now),+ ).fetchone()+ # Keep what a live session still reads; refresh at most every half ttl.+ if row is not None and row[3] - now < self._ttl / 2:+ self._db.execute(+ "UPDATE results SET expires = ? WHERE k = ?", (now + self._ttl, key)+ )+ except sqlite3.Error as exc:+ self._failed("read", exc)+ return None+ return None if row is None else (row[0], row[1], row[2])++ def put(self, key: str, text: str, ratio: float, strategy: str) -> None:+ now = time.time()+ try:+ with self._lock:+ self._db.execute(+ "INSERT OR REPLACE INTO results (k, text, ratio, strategy, expires) "+ "VALUES (?, ?, ?, ?, ?)",+ (key, text, ratio, strategy, now + self._ttl),+ )+ self._writes += 1+ if self._writes % _PURGE_EVERY == 0:+ self._db.execute("DELETE FROM results WHERE expires <= ?", (now,))+ except sqlite3.Error as exc:+ self._failed("write", exc)++ def touch(self, key: str) -> None:+ """Keep ``key`` for another ttl (it is in use from memory)."""+ self._run("UPDATE results SET expires = ? WHERE k = ?", (time.time() + self._ttl, key))++ def delete(self, key: str) -> None:+ self._run("DELETE FROM results WHERE k = ?", (key,))++ def clear_prefix(self, prefix: str) -> None:+ """Delete every row whose key starts with ``prefix`` (a router's namespace)."""+ self._run("DELETE FROM results WHERE substr(k, 1, ?) = ?", (len(prefix), prefix))++ def _run(self, sql: str, args: tuple[object, ...]) -> None:+ try:+ with self._lock:+ self._db.execute(sql, args)+ except sqlite3.Error as exc:+ self._failed("write", exc)++ def _failed(self, what: str, exc: sqlite3.Error) -> None:+ if not self._warned:+ self._warned = True+ logger.warning(+ "compression cache: %s of %s failed (%s); treating it as a miss. "+ "Check the disk, or set HEADROOM_COMPRESSION_CACHE_DISK=0.",+ what,+ self._path,+ exc,+ )+++def open_result_store(ttl_seconds: float) -> ResultStore | None:+ """The disk tier, or ``None`` (memory only) in stateless mode, with+ ``HEADROOM_COMPRESSION_CACHE_DISK=0``, or when the file cannot be opened."""+ from ..paths import persistence_allowed, workspace_dir++ if os.environ.get("HEADROOM_COMPRESSION_CACHE_DISK", "1").strip().lower() in (+ "0",+ "false",+ "no",+ "off",+ ):+ return None+ if not persistence_allowed("compression result cache (compression_cache.db)"):+ return None+ configured = os.environ.get("HEADROOM_COMPRESSION_CACHE_PATH", "").strip()+ path = Path(configured).expanduser() if configured else workspace_dir() / "compression_cache.db"+ try:+ return ResultStore(path, ttl_seconds)+ except (OSError, sqlite3.Error) as exc:+ logger.warning(+ "compression cache: cannot open %s (%s); keeping results in memory only. "+ "Set HEADROOM_COMPRESSION_CACHE_PATH to a writable file or "+ "HEADROOM_COMPRESSION_CACHE_DISK=0.",+ path,+ exc,+ )+ return Nonediff --git a/headroom/proxy/server.py b/headroom/proxy/server.pyindex 36701bda0..59a4c44e2 100644--- a/headroom/proxy/server.py+++ b/headroom/proxy/server.py@@ -1040,6 +1040,10 @@ class HeadroomProxy(# Runs BEFORE the disable_kompress override below so that flag stays# authoritative for turning Kompress off._apply_compressor_selection(router_config, config.compressors)+ # The proxy is long-lived: keep compression results it has no memory+ # budget for in a SQLite file instead of compressing them again+ # (headroom.cache.result_store; off in stateless mode).+ router_config.result_cache_disk = True# External (non-built-in) `headroom.compressor` selections are ignored by# `_apply_compressor_selection` (they have no enable_* flag). Thread them# to the router here so it can route matching blocks through them; Nonediff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex 30552ca09..d8c4c6cd8 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -45,12 +45,13 @@ import reimport sysimport threadingimport time+from collections import OrderedDictfrom collections.abc import Callablefrom concurrent.futures import ThreadPoolExecutorfrom contextvars import ContextVar, copy_contextfrom dataclasses import dataclass, field, replacefrom enum import Enum-from typing import Any+from typing import TYPE_CHECKING, Anyfrom ..config import (DEFAULT_BYTE_EXACT_EXCLUDE_TOOLS,@@ -97,6 +98,9 @@ from .lossless_provider import (from .mixed_content import ContentSection, mixed_content_indicatorsfrom .relevance_split import build_relevance_query, plan_relevance_split, segment+if TYPE_CHECKING:+ from ..cache.result_store import ResultStore+logger = logging.getLogger(__name__)_extract_json_block = _mixed_content._extract_json_block@@ -1558,21 +1562,95 @@ def _netcost_message_tokens(message: dict[str, Any], tokenizer: Tokenizer) -> inreturn count_content_blocks(content, tokenizer.count_text)+#: Default memory budget of the router's compression cache, in MB+#: (``HEADROOM_COMPRESSION_CACHE_MB``).+DEFAULT_COMPRESSION_CACHE_MB = 64+#: Bytes a cache entry costs beyond its text (key, tuple, dict slot).+_CACHE_ENTRY_OVERHEAD = 256+#: How long a compression result is kept after it was last used. Results are+#: mostly one session's tool output, and nothing signals a session's end; a+#: session idle past the provider's longest prompt-cache TTL (Anthropic: 1 h)+#: has no cached prefix left to protect, so compressing it again on resume+#: costs only the compression.+RESULT_TTL_SECONDS = 75 * 60+#: Skip-tier entries kept per MB of budget (each is a key and a timestamp).+_SKIP_ENTRIES_PER_MB = 2000+++def _compression_cache_budget_bytes() -> int:+ raw = os.environ.get("HEADROOM_COMPRESSION_CACHE_MB", "").strip()+ try:+ mb = float(raw) if raw else DEFAULT_COMPRESSION_CACHE_MB+ except ValueError:+ mb = math.nan+ if not math.isfinite(mb) or mb <= 0:+ logger.warning(+ "HEADROOM_COMPRESSION_CACHE_MB=%r is not a positive number of MB; using %d",+ raw,+ DEFAULT_COMPRESSION_CACHE_MB,+ )+ mb = DEFAULT_COMPRESSION_CACHE_MB+ return max(1, int(mb * 1024 * 1024))+++def content_cache_key(content: str, target_ratio: float | None, gated: bool = False) -> str:+ """The compression cache's key for ``content``: a digest, the same in every+ process, so the disk tier (and replicas sharing it) can use it. The runtime+ target ratio is part of it (a different ratio is a different result), and so+ is the reversibility gate tool output passes (#1307)."""+ h = hashlib.blake2b(digest_size=16)+ h.update(f"{target_ratio!r}|{int(gated)}|".encode())+ h.update(content.encode("utf-8", "surrogatepass"))+ return h.hexdigest()+++def _config_namespace(config: Any) -> str:+ """A prefix for this router's keys in the shared disk tier: a digest of its+ config and the headroom version. The proxy may run routers configured+ differently (per-provider Kompress overrides), and a restart may change+ settings; neither may serve output the other made."""+ from dataclasses import asdict++ from .. import __version__++ def stable(value: Any) -> Any:+ # Sets repr in hash-seed order, which differs per process.+ if isinstance(value, (set, frozenset)):+ return sorted(map(stable, value), key=repr)+ if isinstance(value, dict):+ return sorted((repr(k), stable(v)) for k, v in value.items())+ if isinstance(value, (list, tuple)):+ return [stable(v) for v in value]+ return value++ # The router also reads compressor settings from the environment (e.g.+ # HEADROOM_LOSSLESS_THEN_LOSSY) into instance state outside its config;+ # every HEADROOM_* setting is part of the digest, so changing any of them+ # starts a fresh namespace rather than serving output made under another.+ settings = sorted((k, v) for k, v in os.environ.items() if k.startswith("HEADROOM_"))+ h = hashlib.blake2b(digest_size=8)+ h.update(f"{__version__}|{stable(asdict(config))!r}|{settings!r}".encode())+ return h.hexdigest() + ":"++class CompressionCache:- """Two-tier compression cache with TTL. Thread-safe.+ """Two-tier compression cache with idle expiry and a memory budget. Thread-safe.- Tier 1 (skip set): content hashes that won't compress — instant skip,- near-zero memory (just ints in a set).+ Tier 1 (skip set): content keys that won't compress — instant skip.Tier 2 (result cache): compressed results for content that DID compress —reuse the compressed text on subsequent requests.- Entries expire after TTL (default 30min). No max-entries cap — TTL is the- natural bound. Memory grows proportional to compressible content × TTL,- which is bounded by session duration.+ Memory is bounded: results are kept in least-recently-used order within+ ``max_bytes`` (``HEADROOM_COMPRESSION_CACHE_MB``, default 64; results only),+ the skip set within a matching entry count. A result expires ``ttl_seconds``+ after it was last used (a session idle that long has no cached prefix to+ protect). A result carrying a CCR marker expires ``marker_ttl_seconds`` after+ it was made instead, with the CCR original it points to.- Uses in-process dict for ultra-fast lookups (~100ns). Could be backed- by memcached/Redis for multi-process deployments.+ With a ``store`` (the proxy's :class:`~headroom.cache.result_store.ResultStore`),+ marker-free results are also written to disk under ``namespace`` and an+ entry the budget evicted is read back from there instead of compressed again.Thread safety: a ``threading.Lock`` guards all read-modify-writeoperations. The ``apply()`` path runs compression inside a@@ -1581,28 +1659,63 @@ class CompressionCache:wasteful) and metrics counters would drift."""- def __init__(self, ttl_seconds: int = 1800):+ def __init__(+ self,+ ttl_seconds: int = RESULT_TTL_SECONDS,+ max_bytes: int | None = None,+ store: ResultStore | None = None,+ namespace: str = "",+ marker_ttl_seconds: int | None = None,+ ):import threading- # Tier 2: compressed results {hash: (text, ratio, strategy, timestamp)}- self._results: dict[int, tuple[str, float, str, float]] = {}- # Tier 1: hashes of content that won't compress {hash: timestamp}- self._skip: dict[int, float] = {}+ from ..cache.compression_store import _get_env_default_ttl_seconds++ # Tier 2: {key: (text, ratio, strategy, stamp, disk_touched, has_marker)},+ # least recently used first. ``stamp`` is the last use, or the creation+ # time for a result with a CCR marker.+ self._results: OrderedDict[str, tuple[str, float, str, float, float, bool]] = OrderedDict()+ self._bytes = 0+ # Tier 1: {key: marked_at}, oldest first.+ self._skip: OrderedDict[str, float] = OrderedDict()+ self._max_bytes = max_bytes if max_bytes is not None else _compression_cache_budget_bytes()+ self._max_skip = max(1000, self._max_bytes // (1024 * 1024) * _SKIP_ENTRIES_PER_MB)+ self._store = store+ # A namespace of its own, so clear() never reaches other caches' rows.+ self._namespace = namespace or ("cache:" if store is not None else "")+ # {key: sequence numbers of the get()s reading it from disk}. put(),+ # move_to_skip() and clear() drop the key, so a read that raced one of+ # them is not admitted over the newer state; an admitted read drops+ # the earlier ones, which may hold an older row (another worker can+ # write the shared store between them).+ self._disk_reads: dict[str, set[int]] = {}+ self._read_seq = 0# Callbacks invoked (outside the lock) whenever ``clear()`` runs, so- # sibling state keyed by the same content hashes (e.g. the router's+ # sibling state keyed by the same content keys (e.g. the router's# frozen-verdict store) can be reset in lock-step with the cache.self._on_clear: list[Any] = []self._ttl_seconds = ttl_seconds+ self._marker_ttl_seconds = (+ marker_ttl_seconds if marker_ttl_seconds is not None else _get_env_default_ttl_seconds()+ )# Metricsself._hits = 0+ self._disk_hits = 0self._misses = 0self._skip_hits = 0self._evictions = 0self._total_lookup_ns = 0self._lookup_count = 0self._lock = threading.Lock()-- def get(self, key: int) -> tuple[str, float, str] | None:+ # Orders put()'s disk writes, so an older result cannot overwrite or+ # delete the disk row of a newer one for the same key.+ self._disk_lock = threading.Lock()+ # Serializes get()'s disk reads (only memory misses read the disk), so+ # a read numbered later also fetched later and saw a row at least as+ # new. Order: _disk_read_lock, then _lock, then the store's lock.+ self._disk_read_lock = threading.Lock()++ def get(self, key: str) -> tuple[str, float, str] | None:"""Get cached compression result. Thread-safe.Returns (compressed_text, ratio, strategy) or None if not found/expired.@@ -1610,23 +1723,83 @@ class CompressionCache:"""t0 = time.perf_counter_ns()with self._lock:- entry = self._results.get(key)- if entry is not None:- compressed, ratio, strategy, created_at = entry- if (time.monotonic() - created_at) < self._ttl_seconds:- self._hits += 1- self._total_lookup_ns += time.perf_counter_ns() - t0- self._lookup_count += 1- return (compressed, ratio, strategy)+ hit = self._memory_hit_locked(key, t0)+ if hit is not None:+ return hit+ if self._store is None:+ self._misses += 1+ self._record_lookup(t0)+ return None+ with self._disk_read_lock:+ with self._lock:+ # A put() or another read may have admitted it meanwhile.+ hit = self._memory_hit_locked(key, t0)+ if hit is not None:+ return hit+ self._read_seq += 1+ seq = self._read_seq+ self._disk_reads.setdefault(key, set()).add(seq)+ found = self._store.get(self._namespace + key)+ with self._lock:+ readers = self._disk_reads.get(key)+ if readers is not None and seq in readers:+ later = {s for s in readers if s > seq}+ if later:+ self._disk_reads[key] = laterelse:- del self._results[key]- self._evictions += 1- self._misses += 1- self._total_lookup_ns += time.perf_counter_ns() - t0- self._lookup_count += 1+ del self._disk_reads[key]+ if found is not None:+ self._admit_locked(key, *found)+ self._disk_hits += 1+ else:+ # A write for this key, or a later read's admission, landed+ # during the read; the newest result is the one in memory, if any.+ found = self._memory_hit_locked(key, t0)+ if found is not None:+ return found+ if found is None:+ self._misses += 1+ self._record_lookup(t0)+ return found++ def _memory_hit_locked(self, key: str, t0: int) -> tuple[str, float, str] | None:+ """Serve ``key`` from memory if present and unexpired; drop it if expired.++ Every memory read in get() goes through here, so each applies the same+ lifetime contract (marker TTL, idle refresh, disk touch). Caller holds _lock.+ """+ entry = self._results.get(key)+ if entry is None:return None-- def is_skipped(self, key: int) -> bool:+ compressed, ratio, strategy, stamp, disk_touched, has_marker = entry+ now = time.monotonic()+ ttl = self._marker_ttl_seconds if has_marker else self._ttl_seconds+ if (now - stamp) >= ttl:+ self._drop_locked(key)+ self._evictions += 1+ return None+ touch_disk = False+ if not has_marker:+ # A hit keeps the entry for another ttl: expiry counts idle time.+ # The disk copy too, at most every half ttl, so the budget evicting+ # it later cannot lose it mid-session.+ touch_disk = self._store is not None and (now - disk_touched >= ttl / 2)+ self._results[key] = (+ compressed,+ ratio,+ strategy,+ now,+ now if touch_disk else disk_touched,+ False,+ )+ self._results.move_to_end(key)+ self._hits += 1+ self._record_lookup(t0)+ if touch_disk and self._store is not None:+ self._store.touch(self._namespace + key)+ return (compressed, ratio, strategy)++ def is_skipped(self, key: str) -> bool:"""Check if content is known non-compressible (Tier 1). Thread-safe."""with self._lock:ts = self._skip.get(key)@@ -1634,27 +1807,73 @@ class CompressionCache:if (time.monotonic() - ts) < self._ttl_seconds:self._skip_hits += 1return True- else:- del self._skip[key]- self._evictions += 1+ del self._skip[key]+ self._evictions += 1return False- def put(self, key: int, compressed: str, ratio: float, strategy: str) -> None:+ def put(self, key: str, compressed: str, ratio: float, strategy: str) -> None:"""Store a compressed result (Tier 2). Thread-safe."""with self._lock:- self._results[key] = (compressed, ratio, strategy, time.monotonic())+ self._disk_reads.pop(key, None)+ has_marker = self._admit_locked(key, compressed, ratio, strategy)+ if self._store is None:+ return+ with self._disk_lock:+ with self._lock:+ current = self._results.get(key)+ if current is not None and current[0] != compressed:+ return # a later put() replaced it; its disk write wins+ if has_marker:+ # Never kept on disk; nor may an earlier marker-free result for+ # this key come back from there once this one leaves memory.+ self._store.delete(self._namespace + key)+ else:+ self._store.put(self._namespace + key, compressed, ratio, strategy)- def mark_skip(self, key: int) -> None:+ def mark_skip(self, key: str) -> None:"""Mark content as non-compressible (Tier 1). Thread-safe."""with self._lock:- self._skip[key] = time.monotonic()+ self._skip_locked(key)- def move_to_skip(self, key: int) -> None:+ def move_to_skip(self, key: str) -> None:"""Move a result to skip set (threshold tightened, no longer qualifies).Thread-safe."""with self._lock:- self._results.pop(key, None)- self._skip[key] = time.monotonic()+ self._disk_reads.pop(key, None)+ self._drop_locked(key)+ self._skip_locked(key)+ if self._store is not None:+ self._store.delete(self._namespace + key)++ def _admit_locked(self, key: str, compressed: str, ratio: float, strategy: str) -> bool:+ """Add a result, evicting least recently used ones over budget; returns+ whether it carries a CCR marker."""+ self._drop_locked(key)+ now = time.monotonic()+ has_marker = CCR_RETRIEVAL_MARKER_RE.search(compressed) is not None+ self._results[key] = (compressed, ratio, strategy, now, now, has_marker)+ self._bytes += sys.getsizeof(compressed) + _CACHE_ENTRY_OVERHEAD+ # Least recently used first; the entry just added always stays.+ while self._bytes > self._max_bytes and len(self._results) > 1:+ self._drop_locked(next(iter(self._results)))+ self._evictions += 1+ return has_marker++ def _drop_locked(self, key: str) -> None:+ entry = self._results.pop(key, None)+ if entry is not None:+ self._bytes -= sys.getsizeof(entry[0]) + _CACHE_ENTRY_OVERHEAD++ def _skip_locked(self, key: str) -> None:+ self._skip.pop(key, None)+ self._skip[key] = time.monotonic()+ while len(self._skip) > self._max_skip:+ self._skip.popitem(last=False)+ self._evictions += 1++ def _record_lookup(self, t0: int) -> None:+ self._total_lookup_ns += time.perf_counter_ns() - t0+ self._lookup_count += 1@propertydef size(self) -> int:@@ -1672,10 +1891,13 @@ class CompressionCache:avg_ns = self._total_lookup_ns / self._lookup_count if self._lookup_count else 0return {"cache_hits": self._hits,+ "cache_disk_hits": self._disk_hits,"cache_skip_hits": self._skip_hits,"cache_misses": self._misses,"cache_evictions": self._evictions,"cache_size": len(self._results),+ "cache_bytes": self._bytes,+ "cache_max_bytes": self._max_bytes,"cache_skip_size": len(self._skip),"cache_avg_lookup_ns": avg_ns,}@@ -1683,17 +1905,24 @@ class CompressionCache:def register_on_clear(self, callback: Any) -> None:"""Register a callback fired (outside the lock) on every ``clear()``.- Lets state keyed by the same content hashes — e.g. the router's+ Lets state keyed by the same content keys — e.g. the router's``_frozen_verdicts`` store — be reset together with the cache so itcannot outlive the results/skip tiers it shadows."""self._on_clear.append(callback)def clear(self) -> None:- """Clear all entries (e.g., on session end). Thread-safe."""+ """Clear all entries, including this router's rows on disk (other+ workers with the same config share them). Thread-safe."""with self._lock:self._results.clear()+ self._bytes = 0self._skip.clear()+ self._disk_reads.clear()+ # Under the lock, so no disk read can start before the rows are+ # gone and bring a cleared result back into memory.+ if self._store is not None:+ self._store.clear_prefix(self._namespace)# Fire callbacks outside the lock to avoid cross-lock ordering issues.for callback in self._on_clear:callback()@@ -1945,6 +2174,12 @@ class ContentRouterConfig:# emits a `<<ccr:…>>` / `Retrieve …` retrieval marker. SmartCrusher is# additionally forced marker-free via smart_crusher_lossless_only.lossless: bool = False++ # Back the compression cache with a SQLite file (see+ # headroom.cache.result_store) so results its memory budget evicts, or a+ # restart loses, come back without compressing again. The proxy turns it on;+ # library callers keep the cache in memory unless they ask.+ result_cache_disk: bool = False# Cross-turn (whole-conversation) verbatim de-dup. Replaces a contiguous span# in a later tool output that already appeared verbatim in an earlier tool# output with an in-context pointer. Prefix-monotonic (cache-safe) and@@ -2455,7 +2690,12 @@ class ContentRouter(Transform):# are not gated. Now backed by the #3486 ContextVar state — see# the ``_runtime_compression_policy`` property below.- self._cache = CompressionCache()+ store = None+ if self.config.result_cache_disk:+ from ..cache.result_store import open_result_store++ store = open_result_store(ttl_seconds=RESULT_TTL_SECONDS)+ self._cache = CompressionCache(store=store, namespace=_config_namespace(self.config))# Cache-churn fix (HEADROOM_FREEZE_BLOCK_DECISION, default off):# freeze each block's compress-vs-passthrough verdict on first@@ -2473,7 +2713,7 @@ class ContentRouter(Transform):# the parallel compression pass writes verdicts from worker threads, so# we match that posture with a dedicated lock rather than relying on# GIL atomicity (which would not protect the read-then-evict sequence).- self._frozen_verdicts: dict[int, bool] = {}+ self._frozen_verdicts: dict[str, bool] = {}# The store is process-wide (one router per pipeline, shared by every# session), so the cap must scale with the number of CONCURRENT# sessions, not one user's workload: at org scale (many users behind@@ -2636,7 +2876,7 @@ class ContentRouter(Transform):f"preserved_chars~={preserved} (cache bust avoided)")- def _record_frozen_verdict(self, content_key: int, verdict: bool) -> None:+ def _record_frozen_verdict(self, content_key: str, verdict: bool) -> None:"""Record a frozen verdict (thread-safe, bounded FIFO eviction).Caps the store at ``_frozen_verdicts_max`` entries, evicting the@@ -2653,7 +2893,7 @@ class ContentRouter(Transform):del self._frozen_verdicts[oldest]self._frozen_verdicts[content_key] = verdict- def _get_frozen_verdict(self, content_key: int) -> bool | None:+ def _get_frozen_verdict(self, content_key: str) -> bool | None:"""Read a frozen verdict (thread-safe). Returns None if absent."""with self._frozen_lock:return self._frozen_verdicts.get(content_key)@@ -6150,7 +6390,7 @@ class ContentRouter(Transform):# enforce_reversibility, precomputed_detection). The detection is the one# already computed in the routing loop below, threaded through so a# cache-miss message is not re-detected inside compress().- _PendingTask = tuple[int, str, str, float, int, bool, "DetectionResult | None"]+ _PendingTask = tuple[int, str, str, float, str, bool, "DetectionResult | None"]pending_tasks: list[_PendingTask] = []# #856 P2b (flag-gated, default off): net-cost frozen-floor unlock.@@ -6490,13 +6730,13 @@ class ContentRouter(Transform):# Tier 2 (result): known compresses → reuse compressed text.# Key on the runtime target_ratio too: the same content compressed at# a different ratio is a different result, so it must not alias.- content_key = hash((content, getattr(self, "_runtime_target_ratio", None)))# Tool ground truth is gated against lossy-unrecoverable results below# (#1307). Partition its cache namespace so a gated tool entry is never# served from — or poisons — an ungated entry for byte-identical content.enforce_reversibility = role == "tool"- if enforce_reversibility:- content_key = hash((content_key, True))+ content_key = content_cache_key(+ content, getattr(self, "_runtime_target_ratio", None), gated=enforce_reversibility+ )# Tier 1: skip set — instant rejectionif self._cache.is_skipped(content_key):@@ -7560,7 +7800,9 @@ class ContentRouter(Transform):# Two-tier compression cache → shared helpercompressed_content, was_compressed = self._compress_block_content(content=tool_text,- content_key=hash((tool_text, getattr(self, "_runtime_target_ratio", None))),+ content_key=content_cache_key(+ tool_text, getattr(self, "_runtime_target_ratio", None)+ ),context=block_context,bias=bias,min_ratio=min_ratio,@@ -7630,8 +7872,8 @@ class ContentRouter(Transform):# Two-tier compression cache → shared helpercompressed_content, _was_compressed = self._compress_block_content(content=text_content,- content_key=hash(- (text_content, getattr(self, "_runtime_target_ratio", None))+ content_key=content_cache_key(+ text_content, getattr(self, "_runtime_target_ratio", None)),context=context,bias=1.0,@@ -7663,7 +7905,7 @@ class ContentRouter(Transform):def _compress_block_content(self,content: str,- content_key: int,+ content_key: str,context: str,bias: float,min_ratio: float,@@ -7684,7 +7926,7 @@ class ContentRouter(Transform):Args:content: The string content to compress.- content_key: Pre-computed ``hash(content)`` for cache lookups.+ content_key: Pre-computed :func:`content_cache_key` for cache lookups.context: User/query context for relevance-aware compression.bias: Compression bias multiplier (tool-specific or 1.0).min_ratio: Adaptive minimum compression ratio threshold.diff --git a/tests/conftest.py b/tests/conftest.pyindex 23d46bc65..33a8cd19e 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -80,7 +80,7 @@ def _skip_proxy_port_preflight_unless_exercised(@pytest.fixture(autouse=True)-def _scrub_developer_headroom_env(monkeypatch, tmp_path):+def _scrub_developer_headroom_env(monkeypatch, tmp_path, tmp_path_factory):for key in list(os.environ):if key.startswith("HEADROOM_"):monkeypatch.delenv(key, raising=False)@@ -89,6 +89,11 @@ def _scrub_developer_headroom_env(monkeypatch, tmp_path):# test its own store so proxy/CLI startup cannot load developer settings and# saves cannot rewrite them. Tests of path precedence can override this.monkeypatch.setenv("HEADROOM_SETTINGS_PATH", str(tmp_path / "headroom-settings.json"))+ # The proxy's compression cache keeps results on disk; never in the+ # developer's ~/.headroom, never shared between tests, and not in tmp_path,+ # which tests assert stays empty when nothing is persisted.+ cache_dir = tmp_path_factory.mktemp("compression-cache")+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(cache_dir / "compression_cache.db"))# The scrub also deletes the HEADROOM_HARD_WATCHDOG_SECS=0 opt-out CI sets# (#3845), so every test that enters the proxy lifespan armed the 90s# production watchdog inside pytest; it hard-exits the process with code 1diff --git a/tests/test_router_result_cache.py b/tests/test_router_result_cache.pynew file mode 100644index 000000000..2c2d8307a--- /dev/null+++ b/tests/test_router_result_cache.py@@ -0,0 +1,382 @@+"""The content router's compression cache stays within its memory budget."""++from __future__ import annotations++import subprocess+import sys+import threading++from headroom.cache.result_store import ResultStore, open_result_store+from headroom.transforms.content_router import (+ CompressionCache,+ ContentRouter,+ ContentRouterConfig,+ content_cache_key,+)+++def test_results_stay_within_the_memory_budget_least_recently_used_first():+ cache = CompressionCache(max_bytes=10_000)+ for i in range(100):+ cache.put(f"k{i}", "x" * 1000, 0.5, "kompress")+ cache.get("k0") # k0 stays recently used+ assert cache.stats["cache_bytes"] <= 10_000+ assert cache.get("k0") is not None+ assert cache.get("k1") is None # evicted, not kept forever+++def test_an_evicted_result_comes_back_from_disk_unchanged(tmp_path):+ cache = CompressionCache(max_bytes=3_000, store=ResultStore(tmp_path / "c.db", 1800))+ cache.put("first", "compressed once", 0.4, "kompress")+ for i in range(10):+ cache.put(f"k{i}", "y" * 1000, 0.5, "kompress")+ # Out of memory, but read back: no second compression, same bytes.+ assert cache.get("first") == ("compressed once", 0.4, "kompress")+ assert cache.stats["cache_disk_hits"] == 1+++def test_the_skip_set_is_bounded_too():+ cache = CompressionCache(max_bytes=1024 * 1024)+ for i in range(50_000):+ cache.mark_skip(f"s{i}")+ assert cache.skip_size <= 2000+ assert cache.is_skipped("s49999")+++def test_keys_are_the_same_in_every_process():+ code = (+ "from headroom.transforms.content_router import content_cache_key;"+ "print(content_cache_key('same text', 0.5, gated=True))"+ )+ other = subprocess.run(+ [sys.executable, "-c", code], capture_output=True, text=True, check=True+ ).stdout.strip()+ assert other == content_cache_key("same text", 0.5, gated=True)+ assert content_cache_key("same text", 0.5) != other # gated tool output kept apart+++def test_stateless_mode_writes_no_cache_file(tmp_path, monkeypatch):+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))+ monkeypatch.setenv("HEADROOM_STATELESS", "1")+ assert open_result_store(1800) is None+ assert not (tmp_path / "compression_cache.db").exists()+++def test_a_result_in_use_is_kept_and_an_idle_one_is_dropped(tmp_path, monkeypatch):+ # Nothing signals a session's end: a result expires once unused for the+ # ttl, from memory and from disk, while a session still reading it keeps it.+ import time as time_module++ clock = [1000.0]+ monkeypatch.setattr(time_module, "monotonic", lambda: clock[0])+ monkeypatch.setattr(time_module, "time", lambda: clock[0])+ store = ResultStore(tmp_path / "c.db", ttl_seconds=100)+ cache = CompressionCache(ttl_seconds=100, store=store)+ cache.put("live", "kept", 0.5, "kompress")+ cache.put("idle", "dropped", 0.5, "kompress")+ for _ in range(5): # 'live' is read every 60 s, for 300 s+ clock[0] += 60+ assert cache.get("live") is not None+ assert cache.get("idle") is None+ assert store.get(cache._namespace + "idle") is None+ assert store.get(cache._namespace + "live") == ("kept", 0.5, "kompress")+++def test_routers_configured_differently_never_share_output(tmp_path, monkeypatch):+ # The proxy can run two routers (per-provider Kompress overrides), and a+ # restart can change settings; both read the same file.+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "c.db"))+ lossy = ContentRouter(ContentRouterConfig(result_cache_disk=True))+ lossless = ContentRouter(ContentRouterConfig(result_cache_disk=True, lossless=True))+ lossy._cache.put("k", "lossy output", 0.3, "kompress")+ assert lossless._cache.get("k") is None+ same = ContentRouter(ContentRouterConfig(result_cache_disk=True))+ assert same._cache.get("k") == ("lossy output", 0.3, "kompress")+++def test_the_namespace_is_the_same_in_every_process():+ code = (+ "from headroom.transforms.content_router import ContentRouterConfig, _config_namespace;"+ "print(_config_namespace(ContentRouterConfig()))"+ )+ seen = {+ subprocess.run(+ [sys.executable, "-c", code],+ capture_output=True,+ text=True,+ check=True,+ env={**__import__("os").environ, "PYTHONHASHSEED": seed},+ ).stdout.strip()+ for seed in ("1", "2")+ }+ assert len(seen) == 1+++def test_a_broken_database_is_a_miss_not_a_failed_request(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(store=store)+ store._db.close()+ cache.put("k", "text", 0.5, "kompress") # no raise+ cache.clear()+ assert cache.get("k") is None+++def test_a_result_pointing_at_a_ccr_original_is_not_kept_on_disk(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(store=store)+ cache.put("k", "first lines <<ccr:0123456789abcdef>>", 0.2, "x")+ assert store.get(cache._namespace + "k") is None+++def test_a_nonfinite_budget_falls_back_instead_of_stopping_startup(monkeypatch):+ for raw in ("nan", "inf", "-5"):+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_MB", raw)+ assert CompressionCache().stats["cache_max_bytes"] == 64 * 1024 * 1024+++def test_a_changed_compressor_setting_does_not_reuse_old_results(tmp_path, monkeypatch):+ # Settings the router reads from the environment, outside its config.+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "c.db"))+ before = ContentRouter(ContentRouterConfig(result_cache_disk=True))+ before._cache.put("k", "made under the old setting", 0.3, "kompress")+ monkeypatch.setenv("HEADROOM_LOSSLESS_THEN_LOSSY", "1")+ after = ContentRouter(ContentRouterConfig(result_cache_disk=True))+ assert after._cache.get("k") is None+++def test_a_marker_result_replacing_a_plain_one_removes_it_from_disk(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=4_000, store=store)+ cache.put("k", "plain result", 0.5, "kompress")+ cache.put("k", "first lines <<ccr:0123456789abcdef>>", 0.2, "x")+ for i in range(10): # push "k" out of memory+ cache.put(f"filler{i}", "f" * 1000, 0.5, "kompress")+ assert cache.get("k") is None+++def test_clear_also_clears_the_routers_rows_on_disk(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(store=store, namespace="ns1:")+ other = CompressionCache(store=store, namespace="ns2:")+ cache.put("k", "value", 0.5, "kompress")+ other.put("k", "theirs", 0.5, "kompress")+ cache.clear()+ assert cache.get("k") is None+ assert store.get("ns2:k") == ("theirs", 0.5, "kompress")+++def test_an_older_put_never_undoes_a_newer_ones_disk_row(tmp_path):+ # A marker put admitted first but reaching the disk after a newer plain+ # put must not delete the plain result's row.+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(store=store)++ class NewerPutRunsFirst:+ def __init__(self) -> None:+ self._lock, self._pending = threading.Lock(), True++ def __enter__(self):+ if self._pending: # between the marker's admit and its disk step+ self._pending = False+ cache.put("k", "plain output", 0.5, "text")+ self._lock.acquire()++ def __exit__(self, *exc):+ self._lock.release()++ cache._disk_lock = NewerPutRunsFirst()+ cache.put("k", "<<ccr:0123456789abcdef 3_rows_offloaded>>", 0.5, "text")+ assert store.get(cache._namespace + "k")[0] == "plain output"+++def test_a_cache_without_a_namespace_clears_only_its_own_rows(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ mine = CompressionCache(store=store)+ other = CompressionCache(store=store, namespace="router-b:")+ mine.put("a", "result a", 0.5, "text")+ other.put("b", "result b", 0.5, "text")+ mine.clear()+ assert store.get("router-b:b") is not None+++def test_a_disk_read_in_flight_never_replaces_a_newer_put(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=600, store=store)+ cache.put("k", "old", 0.5, "text")+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory+ read_done, release = threading.Event(), threading.Event()+ real_get = store.get++ def paused_get(key):+ found = real_get(key)+ store.get = real_get # pause only the first read+ read_done.set()+ assert release.wait(5)+ return found++ store.get = paused_get+ results = []+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))+ reader.start()+ assert read_done.wait(5) # the read has "old" in hand+ cache.put("k", "new", 0.5, "text")+ release.set()+ reader.join(5)+ assert results == [("new", 0.5, "text")]+ assert cache.get("k") == ("new", 0.5, "text")+ assert store.get(cache._namespace + "k")[0] == "new"+++def test_a_disk_read_during_clear_never_restores_a_cleared_result(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=600, store=store)+ cache.put("k", "old", 0.5, "text")+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory+ deleting, release = threading.Event(), threading.Event()+ real_clear_prefix = store.clear_prefix++ def paused_clear_prefix(prefix):+ deleting.set()+ assert release.wait(5)+ real_clear_prefix(prefix)++ store.clear_prefix = paused_clear_prefix+ clearer = threading.Thread(target=cache.clear)+ clearer.start()+ assert deleting.wait(5) # memory cleared, disk row not yet deleted+ reader = threading.Thread(target=cache.get, args=("k",))+ reader.start()+ reader.join(0.5) # finishes here unless clear() still holds it off+ release.set()+ clearer.join(5)+ reader.join(5)+ assert store.get(cache._namespace + "k") is None+ assert cache.get("k") is None+++class PauseAfterFetch:+ """Stands in for a cache's ``_disk_read_lock``: the first ``pauses`` reads+ stop after their fetch, outside the lock and before admission."""++ def __init__(self, pauses: int) -> None:+ self._lock = threading.Lock()+ self.fetched = [threading.Event() for _ in range(pauses)]+ self.release = [threading.Event() for _ in range(pauses)]+ self._exits = 0++ def __enter__(self):+ self._lock.acquire()++ def __exit__(self, *exc):+ i, self._exits = self._exits, self._exits + 1+ self._lock.release()+ if i < len(self.fetched):+ self.fetched[i].set()+ assert self.release[i].wait(5)+++def test_concurrent_disk_reads_of_one_key_each_return_the_result(tmp_path):+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=600, store=store)+ cache.put("k", "kept", 0.5, "text")+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory+ reads = cache._disk_read_lock = PauseAfterFetch(2)+ results: dict[str, object] = {}+ first = threading.Thread(target=lambda: results.update(first=cache.get("k")))+ second = threading.Thread(target=lambda: results.update(second=cache.get("k")))+ first.start()+ assert reads.fetched[0].wait(5)+ second.start()+ assert reads.fetched[1].wait(5) # both reads in flight; the first finishes first+ reads.release[0].set()+ first.join(5)+ reads.release[1].set()+ second.join(5)+ assert results == {"first": ("kept", 0.5, "text"), "second": ("kept", 0.5, "text")}+++def test_an_older_disk_read_never_replaces_a_later_reads_result(tmp_path):+ # Two workers share one store: a read that fetched earlier holds an older+ # row than one that fetched after the other worker's put.+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=600, store=store)+ other_worker = CompressionCache(store=store)+ cache.put("k", "old", 0.5, "text")+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory+ reads = cache._disk_read_lock = PauseAfterFetch(1)+ results = []+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))+ reader.start()+ assert reads.fetched[0].wait(5) # the first read has "old" in hand+ other_worker.put("k", "new", 0.5, "text")+ assert cache.get("k") == ("new", 0.5, "text") # a later read admits "new"+ reads.release[0].set()+ reader.join(5)+ assert results == [("new", 0.5, "text")]+ assert cache.get("k") == ("new", 0.5, "text")+++def test_a_read_that_fetches_later_is_never_outranked_by_an_older_row(tmp_path):+ # Read A is numbered before read B but fetches after another worker's+ # put: what A fetched is the newest row and must be what memory keeps.+ store = ResultStore(tmp_path / "c.db", 1800)+ cache = CompressionCache(max_bytes=600, store=store)+ other_worker = CompressionCache(store=store)+ cache.put("k", "old", 0.5, "text")+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory+ fetching, release = threading.Event(), threading.Event()+ real_get = store.get++ def paused_get(key):+ store.get = real_get # pause only the first read, before it fetches+ fetching.set()+ assert release.wait(5)+ return real_get(key)++ store.get = paused_get+ results = []+ first = threading.Thread(target=lambda: results.append(cache.get("k")))+ first.start()+ assert fetching.wait(5) # A is numbered; its fetch has not run+ second = threading.Thread(target=cache.get, args=("k",))+ second.start()+ second.join(0.5) # B reads "old" here unless A's read holds it off+ other_worker.put("k", "new", 0.5, "text")+ release.set()+ first.join(5)+ second.join(5)+ assert results == [("new", 0.5, "text")]+ assert cache.get("k") == ("new", 0.5, "text")+ assert store.get(cache._namespace + "k")[0] == "new"+++def test_a_result_that_expires_while_a_read_waits_is_not_returned(tmp_path, monkeypatch):+ # get() misses memory, then waits for the disk-read lock; a marker result+ # admitted meanwhile expires before the wait ends. Its second look at+ # memory must apply the same lifetime as the first, not return it.+ import time as time_module++ clock = [1000.0]+ monkeypatch.setattr(time_module, "monotonic", lambda: clock[0])+ cache = CompressionCache(store=ResultStore(tmp_path / "c.db", 1800), marker_ttl_seconds=1)+ waiting, real = threading.Event(), cache._disk_read_lock++ class SignalsWait:+ def __enter__(self):+ waiting.set()+ return real.__enter__()++ def __exit__(self, *exc):+ return real.__exit__(*exc)++ cache._disk_read_lock = SignalsWait()+ results = []+ with real:+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))+ reader.start()+ assert waiting.wait(5)+ cache.put("k", "head <<ccr:0123456789abcdef>>", 0.2, "x")+ clock[0] += 2+ reader.join(5)+ assert results == [None]+ assert cache.get("k") is None