Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): bound the router's compression cache, keep the rest in SQLite (#4357)

Tejas Choprab196424e35df
diff --git a/headroom/cache/result_store.py b/headroom/cache/result_store.py
new file mode 100644
index 000000000..0ec978eb8
--- /dev/null
+++ b/headroom/cache/result_store.py
@@ -0,0 +1,159 @@
+"""SQLite tier behind the content router's in-memory compression cache.
+
+The router keeps recent results in a byte-budgeted LRU; this file holds the rest,
+so an entry the budget evicted comes back from disk instead of being compressed
+again (which costs a Kompress pass and can give different bytes, rewriting a
+cached prompt prefix). It also survives a restart and is shared by worker
+processes (WAL), like the CCR store; WAL needs a local filesystem.
+
+Each router reads and writes under its own namespace (a digest of its config and
+the headroom version), so routers configured differently, or a restart with
+other settings, never serve each other's output. Results carrying a CCR marker
+are not written here: their originals live in the CCR store, on its own clock.
+
+The file holds compressed tool output, so it follows the CCR store's rules:
+owner-only, ``workspace_dir()/compression_cache.db`` (``HEADROOM_COMPRESSION_CACHE_PATH``
+to move it), expired rows deleted on open, never opened in stateless mode, and
+``HEADROOM_COMPRESSION_CACHE_DISK=0`` keeps the cache in memory only. A database
+error is a miss, never a failed request.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+from ..fileperms import connect_private_sqlite
+
+logger = logging.getLogger(__name__)
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS results (
+ k TEXT PRIMARY KEY,
+ text TEXT NOT NULL,
+ ratio REAL NOT NULL,
+ strategy TEXT NOT NULL,
+ expires REAL NOT NULL
+);
+CREATE INDEX IF NOT EXISTS idx_results_expires ON results (expires);
+"""
+#: Expired rows are deleted every this many writes, and on open.
+_PURGE_EVERY = 1000
+
+
+class ResultStore:
+ """Compressed results by key; an entry expires ``ttl_seconds`` after it was
+ last read or written. Thread-safe; several processes may share the file."""
+
+ def __init__(self, path: Path, ttl_seconds: float) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ self._path = path
+ self._ttl = ttl_seconds
+ self._lock = threading.Lock()
+ self._writes = 0
+ self._warned = False
+ self._db = connect_private_sqlite(
+ path, what="compression cache", check_same_thread=False, isolation_level=None
+ )
+ self._db.execute("PRAGMA busy_timeout=5000")
+ self._db.execute("PRAGMA journal_mode=WAL")
+ self._db.execute("PRAGMA synchronous=NORMAL")
+ self._db.executescript(_SCHEMA)
+ # Sessions that ended while the proxy was down go now.
+ self._db.execute("DELETE FROM results WHERE expires <= ?", (time.time(),))
+
+ def get(self, key: str) -> tuple[str, float, str] | None:
+ now = time.time()
+ try:
+ with self._lock:
+ row = self._db.execute(
+ "SELECT text, ratio, strategy, expires FROM results "
+ "WHERE k = ? AND expires > ?",
+ (key, now),
+ ).fetchone()
+ # Keep what a live session still reads; refresh at most every half ttl.
+ if row is not None and row[3] - now < self._ttl / 2:
+ self._db.execute(
+ "UPDATE results SET expires = ? WHERE k = ?", (now + self._ttl, key)
+ )
+ except sqlite3.Error as exc:
+ self._failed("read", exc)
+ return None
+ return None if row is None else (row[0], row[1], row[2])
+
+ def put(self, key: str, text: str, ratio: float, strategy: str) -> None:
+ now = time.time()
+ try:
+ with self._lock:
+ self._db.execute(
+ "INSERT OR REPLACE INTO results (k, text, ratio, strategy, expires) "
+ "VALUES (?, ?, ?, ?, ?)",
+ (key, text, ratio, strategy, now + self._ttl),
+ )
+ self._writes += 1
+ if self._writes % _PURGE_EVERY == 0:
+ self._db.execute("DELETE FROM results WHERE expires <= ?", (now,))
+ except sqlite3.Error as exc:
+ self._failed("write", exc)
+
+ def touch(self, key: str) -> None:
+ """Keep ``key`` for another ttl (it is in use from memory)."""
+ self._run("UPDATE results SET expires = ? WHERE k = ?", (time.time() + self._ttl, key))
+
+ def delete(self, key: str) -> None:
+ self._run("DELETE FROM results WHERE k = ?", (key,))
+
+ def clear_prefix(self, prefix: str) -> None:
+ """Delete every row whose key starts with ``prefix`` (a router's namespace)."""
+ self._run("DELETE FROM results WHERE substr(k, 1, ?) = ?", (len(prefix), prefix))
+
+ def _run(self, sql: str, args: tuple[object, ...]) -> None:
+ try:
+ with self._lock:
+ self._db.execute(sql, args)
+ except sqlite3.Error as exc:
+ self._failed("write", exc)
+
+ def _failed(self, what: str, exc: sqlite3.Error) -> None:
+ if not self._warned:
+ self._warned = True
+ logger.warning(
+ "compression cache: %s of %s failed (%s); treating it as a miss. "
+ "Check the disk, or set HEADROOM_COMPRESSION_CACHE_DISK=0.",
+ what,
+ self._path,
+ exc,
+ )
+
+
+def open_result_store(ttl_seconds: float) -> ResultStore | None:
+ """The disk tier, or ``None`` (memory only) in stateless mode, with
+ ``HEADROOM_COMPRESSION_CACHE_DISK=0``, or when the file cannot be opened."""
+ from ..paths import persistence_allowed, workspace_dir
+
+ if os.environ.get("HEADROOM_COMPRESSION_CACHE_DISK", "1").strip().lower() in (
+ "0",
+ "false",
+ "no",
+ "off",
+ ):
+ return None
+ if not persistence_allowed("compression result cache (compression_cache.db)"):
+ return None
+ configured = os.environ.get("HEADROOM_COMPRESSION_CACHE_PATH", "").strip()
+ path = Path(configured).expanduser() if configured else workspace_dir() / "compression_cache.db"
+ try:
+ return ResultStore(path, ttl_seconds)
+ except (OSError, sqlite3.Error) as exc:
+ logger.warning(
+ "compression cache: cannot open %s (%s); keeping results in memory only. "
+ "Set HEADROOM_COMPRESSION_CACHE_PATH to a writable file or "
+ "HEADROOM_COMPRESSION_CACHE_DISK=0.",
+ path,
+ exc,
+ )
+ return None
diff --git a/headroom/proxy/server.py b/headroom/proxy/server.py
index 36701bda0..59a4c44e2 100644
--- a/headroom/proxy/server.py
+++ b/headroom/proxy/server.py
@@ -1040,6 +1040,10 @@ class HeadroomProxy(
# Runs BEFORE the disable_kompress override below so that flag stays
# authoritative for turning Kompress off.
_apply_compressor_selection(router_config, config.compressors)
+ # The proxy is long-lived: keep compression results it has no memory
+ # budget for in a SQLite file instead of compressing them again
+ # (headroom.cache.result_store; off in stateless mode).
+ router_config.result_cache_disk = True
# External (non-built-in) `headroom.compressor` selections are ignored by
# `_apply_compressor_selection` (they have no enable_* flag). Thread them
# to the router here so it can route matching blocks through them; None
diff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.py
index 30552ca09..d8c4c6cd8 100644
--- a/headroom/transforms/content_router.py
+++ b/headroom/transforms/content_router.py
@@ -45,12 +45,13 @@ import re
import sys
import threading
import time
+from collections import OrderedDict
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field, replace
from enum import Enum
-from typing import Any
+from typing import TYPE_CHECKING, Any
from ..config import (
DEFAULT_BYTE_EXACT_EXCLUDE_TOOLS,
@@ -97,6 +98,9 @@ from .lossless_provider import (
from .mixed_content import ContentSection, mixed_content_indicators
from .relevance_split import build_relevance_query, plan_relevance_split, segment
+if TYPE_CHECKING:
+ from ..cache.result_store import ResultStore
+
logger = logging.getLogger(__name__)
_extract_json_block = _mixed_content._extract_json_block
@@ -1558,21 +1562,95 @@ def _netcost_message_tokens(message: dict[str, Any], tokenizer: Tokenizer) -> in
return count_content_blocks(content, tokenizer.count_text)
+#: Default memory budget of the router's compression cache, in MB
+#: (``HEADROOM_COMPRESSION_CACHE_MB``).
+DEFAULT_COMPRESSION_CACHE_MB = 64
+#: Bytes a cache entry costs beyond its text (key, tuple, dict slot).
+_CACHE_ENTRY_OVERHEAD = 256
+#: How long a compression result is kept after it was last used. Results are
+#: mostly one session's tool output, and nothing signals a session's end; a
+#: session idle past the provider's longest prompt-cache TTL (Anthropic: 1 h)
+#: has no cached prefix left to protect, so compressing it again on resume
+#: costs only the compression.
+RESULT_TTL_SECONDS = 75 * 60
+#: Skip-tier entries kept per MB of budget (each is a key and a timestamp).
+_SKIP_ENTRIES_PER_MB = 2000
+
+
+def _compression_cache_budget_bytes() -> int:
+ raw = os.environ.get("HEADROOM_COMPRESSION_CACHE_MB", "").strip()
+ try:
+ mb = float(raw) if raw else DEFAULT_COMPRESSION_CACHE_MB
+ except ValueError:
+ mb = math.nan
+ if not math.isfinite(mb) or mb <= 0:
+ logger.warning(
+ "HEADROOM_COMPRESSION_CACHE_MB=%r is not a positive number of MB; using %d",
+ raw,
+ DEFAULT_COMPRESSION_CACHE_MB,
+ )
+ mb = DEFAULT_COMPRESSION_CACHE_MB
+ return max(1, int(mb * 1024 * 1024))
+
+
+def content_cache_key(content: str, target_ratio: float | None, gated: bool = False) -> str:
+ """The compression cache's key for ``content``: a digest, the same in every
+ process, so the disk tier (and replicas sharing it) can use it. The runtime
+ target ratio is part of it (a different ratio is a different result), and so
+ is the reversibility gate tool output passes (#1307)."""
+ h = hashlib.blake2b(digest_size=16)
+ h.update(f"{target_ratio!r}|{int(gated)}|".encode())
+ h.update(content.encode("utf-8", "surrogatepass"))
+ return h.hexdigest()
+
+
+def _config_namespace(config: Any) -> str:
+ """A prefix for this router's keys in the shared disk tier: a digest of its
+ config and the headroom version. The proxy may run routers configured
+ differently (per-provider Kompress overrides), and a restart may change
+ settings; neither may serve output the other made."""
+ from dataclasses import asdict
+
+ from .. import __version__
+
+ def stable(value: Any) -> Any:
+ # Sets repr in hash-seed order, which differs per process.
+ if isinstance(value, (set, frozenset)):
+ return sorted(map(stable, value), key=repr)
+ if isinstance(value, dict):
+ return sorted((repr(k), stable(v)) for k, v in value.items())
+ if isinstance(value, (list, tuple)):
+ return [stable(v) for v in value]
+ return value
+
+ # The router also reads compressor settings from the environment (e.g.
+ # HEADROOM_LOSSLESS_THEN_LOSSY) into instance state outside its config;
+ # every HEADROOM_* setting is part of the digest, so changing any of them
+ # starts a fresh namespace rather than serving output made under another.
+ settings = sorted((k, v) for k, v in os.environ.items() if k.startswith("HEADROOM_"))
+ h = hashlib.blake2b(digest_size=8)
+ h.update(f"{__version__}|{stable(asdict(config))!r}|{settings!r}".encode())
+ return h.hexdigest() + ":"
+
+
class CompressionCache:
- """Two-tier compression cache with TTL. Thread-safe.
+ """Two-tier compression cache with idle expiry and a memory budget. Thread-safe.
- Tier 1 (skip set): content hashes that won't compress — instant skip,
- near-zero memory (just ints in a set).
+ Tier 1 (skip set): content keys that won't compress — instant skip.
Tier 2 (result cache): compressed results for content that DID compress —
reuse the compressed text on subsequent requests.
- Entries expire after TTL (default 30min). No max-entries cap — TTL is the
- natural bound. Memory grows proportional to compressible content × TTL,
- which is bounded by session duration.
+ Memory is bounded: results are kept in least-recently-used order within
+ ``max_bytes`` (``HEADROOM_COMPRESSION_CACHE_MB``, default 64; results only),
+ the skip set within a matching entry count. A result expires ``ttl_seconds``
+ after it was last used (a session idle that long has no cached prefix to
+ protect). A result carrying a CCR marker expires ``marker_ttl_seconds`` after
+ it was made instead, with the CCR original it points to.
- Uses in-process dict for ultra-fast lookups (~100ns). Could be backed
- by memcached/Redis for multi-process deployments.
+ With a ``store`` (the proxy's :class:`~headroom.cache.result_store.ResultStore`),
+ marker-free results are also written to disk under ``namespace`` and an
+ entry the budget evicted is read back from there instead of compressed again.
Thread safety: a ``threading.Lock`` guards all read-modify-write
operations. The ``apply()`` path runs compression inside a
@@ -1581,28 +1659,63 @@ class CompressionCache:
wasteful) and metrics counters would drift.
"""
- def __init__(self, ttl_seconds: int = 1800):
+ def __init__(
+ self,
+ ttl_seconds: int = RESULT_TTL_SECONDS,
+ max_bytes: int | None = None,
+ store: ResultStore | None = None,
+ namespace: str = "",
+ marker_ttl_seconds: int | None = None,
+ ):
import threading
- # Tier 2: compressed results {hash: (text, ratio, strategy, timestamp)}
- self._results: dict[int, tuple[str, float, str, float]] = {}
- # Tier 1: hashes of content that won't compress {hash: timestamp}
- self._skip: dict[int, float] = {}
+ from ..cache.compression_store import _get_env_default_ttl_seconds
+
+ # Tier 2: {key: (text, ratio, strategy, stamp, disk_touched, has_marker)},
+ # least recently used first. ``stamp`` is the last use, or the creation
+ # time for a result with a CCR marker.
+ self._results: OrderedDict[str, tuple[str, float, str, float, float, bool]] = OrderedDict()
+ self._bytes = 0
+ # Tier 1: {key: marked_at}, oldest first.
+ self._skip: OrderedDict[str, float] = OrderedDict()
+ self._max_bytes = max_bytes if max_bytes is not None else _compression_cache_budget_bytes()
+ self._max_skip = max(1000, self._max_bytes // (1024 * 1024) * _SKIP_ENTRIES_PER_MB)
+ self._store = store
+ # A namespace of its own, so clear() never reaches other caches' rows.
+ self._namespace = namespace or ("cache:" if store is not None else "")
+ # {key: sequence numbers of the get()s reading it from disk}. put(),
+ # move_to_skip() and clear() drop the key, so a read that raced one of
+ # them is not admitted over the newer state; an admitted read drops
+ # the earlier ones, which may hold an older row (another worker can
+ # write the shared store between them).
+ self._disk_reads: dict[str, set[int]] = {}
+ self._read_seq = 0
# Callbacks invoked (outside the lock) whenever ``clear()`` runs, so
- # sibling state keyed by the same content hashes (e.g. the router's
+ # sibling state keyed by the same content keys (e.g. the router's
# frozen-verdict store) can be reset in lock-step with the cache.
self._on_clear: list[Any] = []
self._ttl_seconds = ttl_seconds
+ self._marker_ttl_seconds = (
+ marker_ttl_seconds if marker_ttl_seconds is not None else _get_env_default_ttl_seconds()
+ )
# Metrics
self._hits = 0
+ self._disk_hits = 0
self._misses = 0
self._skip_hits = 0
self._evictions = 0
self._total_lookup_ns = 0
self._lookup_count = 0
self._lock = threading.Lock()
-
- def get(self, key: int) -> tuple[str, float, str] | None:
+ # Orders put()'s disk writes, so an older result cannot overwrite or
+ # delete the disk row of a newer one for the same key.
+ self._disk_lock = threading.Lock()
+ # Serializes get()'s disk reads (only memory misses read the disk), so
+ # a read numbered later also fetched later and saw a row at least as
+ # new. Order: _disk_read_lock, then _lock, then the store's lock.
+ self._disk_read_lock = threading.Lock()
+
+ def get(self, key: str) -> tuple[str, float, str] | None:
"""Get cached compression result. Thread-safe.
Returns (compressed_text, ratio, strategy) or None if not found/expired.
@@ -1610,23 +1723,83 @@ class CompressionCache:
"""
t0 = time.perf_counter_ns()
with self._lock:
- entry = self._results.get(key)
- if entry is not None:
- compressed, ratio, strategy, created_at = entry
- if (time.monotonic() - created_at) < self._ttl_seconds:
- self._hits += 1
- self._total_lookup_ns += time.perf_counter_ns() - t0
- self._lookup_count += 1
- return (compressed, ratio, strategy)
+ hit = self._memory_hit_locked(key, t0)
+ if hit is not None:
+ return hit
+ if self._store is None:
+ self._misses += 1
+ self._record_lookup(t0)
+ return None
+ with self._disk_read_lock:
+ with self._lock:
+ # A put() or another read may have admitted it meanwhile.
+ hit = self._memory_hit_locked(key, t0)
+ if hit is not None:
+ return hit
+ self._read_seq += 1
+ seq = self._read_seq
+ self._disk_reads.setdefault(key, set()).add(seq)
+ found = self._store.get(self._namespace + key)
+ with self._lock:
+ readers = self._disk_reads.get(key)
+ if readers is not None and seq in readers:
+ later = {s for s in readers if s > seq}
+ if later:
+ self._disk_reads[key] = later
else:
- del self._results[key]
- self._evictions += 1
- self._misses += 1
- self._total_lookup_ns += time.perf_counter_ns() - t0
- self._lookup_count += 1
+ del self._disk_reads[key]
+ if found is not None:
+ self._admit_locked(key, *found)
+ self._disk_hits += 1
+ else:
+ # A write for this key, or a later read's admission, landed
+ # during the read; the newest result is the one in memory, if any.
+ found = self._memory_hit_locked(key, t0)
+ if found is not None:
+ return found
+ if found is None:
+ self._misses += 1
+ self._record_lookup(t0)
+ return found
+
+ def _memory_hit_locked(self, key: str, t0: int) -> tuple[str, float, str] | None:
+ """Serve ``key`` from memory if present and unexpired; drop it if expired.
+
+ Every memory read in get() goes through here, so each applies the same
+ lifetime contract (marker TTL, idle refresh, disk touch). Caller holds _lock.
+ """
+ entry = self._results.get(key)
+ if entry is None:
return None
-
- def is_skipped(self, key: int) -> bool:
+ compressed, ratio, strategy, stamp, disk_touched, has_marker = entry
+ now = time.monotonic()
+ ttl = self._marker_ttl_seconds if has_marker else self._ttl_seconds
+ if (now - stamp) >= ttl:
+ self._drop_locked(key)
+ self._evictions += 1
+ return None
+ touch_disk = False
+ if not has_marker:
+ # A hit keeps the entry for another ttl: expiry counts idle time.
+ # The disk copy too, at most every half ttl, so the budget evicting
+ # it later cannot lose it mid-session.
+ touch_disk = self._store is not None and (now - disk_touched >= ttl / 2)
+ self._results[key] = (
+ compressed,
+ ratio,
+ strategy,
+ now,
+ now if touch_disk else disk_touched,
+ False,
+ )
+ self._results.move_to_end(key)
+ self._hits += 1
+ self._record_lookup(t0)
+ if touch_disk and self._store is not None:
+ self._store.touch(self._namespace + key)
+ return (compressed, ratio, strategy)
+
+ def is_skipped(self, key: str) -> bool:
"""Check if content is known non-compressible (Tier 1). Thread-safe."""
with self._lock:
ts = self._skip.get(key)
@@ -1634,27 +1807,73 @@ class CompressionCache:
if (time.monotonic() - ts) < self._ttl_seconds:
self._skip_hits += 1
return True
- else:
- del self._skip[key]
- self._evictions += 1
+ del self._skip[key]
+ self._evictions += 1
return False
- def put(self, key: int, compressed: str, ratio: float, strategy: str) -> None:
+ def put(self, key: str, compressed: str, ratio: float, strategy: str) -> None:
"""Store a compressed result (Tier 2). Thread-safe."""
with self._lock:
- self._results[key] = (compressed, ratio, strategy, time.monotonic())
+ self._disk_reads.pop(key, None)
+ has_marker = self._admit_locked(key, compressed, ratio, strategy)
+ if self._store is None:
+ return
+ with self._disk_lock:
+ with self._lock:
+ current = self._results.get(key)
+ if current is not None and current[0] != compressed:
+ return # a later put() replaced it; its disk write wins
+ if has_marker:
+ # Never kept on disk; nor may an earlier marker-free result for
+ # this key come back from there once this one leaves memory.
+ self._store.delete(self._namespace + key)
+ else:
+ self._store.put(self._namespace + key, compressed, ratio, strategy)
- def mark_skip(self, key: int) -> None:
+ def mark_skip(self, key: str) -> None:
"""Mark content as non-compressible (Tier 1). Thread-safe."""
with self._lock:
- self._skip[key] = time.monotonic()
+ self._skip_locked(key)
- def move_to_skip(self, key: int) -> None:
+ def move_to_skip(self, key: str) -> None:
"""Move a result to skip set (threshold tightened, no longer qualifies).
Thread-safe."""
with self._lock:
- self._results.pop(key, None)
- self._skip[key] = time.monotonic()
+ self._disk_reads.pop(key, None)
+ self._drop_locked(key)
+ self._skip_locked(key)
+ if self._store is not None:
+ self._store.delete(self._namespace + key)
+
+ def _admit_locked(self, key: str, compressed: str, ratio: float, strategy: str) -> bool:
+ """Add a result, evicting least recently used ones over budget; returns
+ whether it carries a CCR marker."""
+ self._drop_locked(key)
+ now = time.monotonic()
+ has_marker = CCR_RETRIEVAL_MARKER_RE.search(compressed) is not None
+ self._results[key] = (compressed, ratio, strategy, now, now, has_marker)
+ self._bytes += sys.getsizeof(compressed) + _CACHE_ENTRY_OVERHEAD
+ # Least recently used first; the entry just added always stays.
+ while self._bytes > self._max_bytes and len(self._results) > 1:
+ self._drop_locked(next(iter(self._results)))
+ self._evictions += 1
+ return has_marker
+
+ def _drop_locked(self, key: str) -> None:
+ entry = self._results.pop(key, None)
+ if entry is not None:
+ self._bytes -= sys.getsizeof(entry[0]) + _CACHE_ENTRY_OVERHEAD
+
+ def _skip_locked(self, key: str) -> None:
+ self._skip.pop(key, None)
+ self._skip[key] = time.monotonic()
+ while len(self._skip) > self._max_skip:
+ self._skip.popitem(last=False)
+ self._evictions += 1
+
+ def _record_lookup(self, t0: int) -> None:
+ self._total_lookup_ns += time.perf_counter_ns() - t0
+ self._lookup_count += 1
@property
def size(self) -> int:
@@ -1672,10 +1891,13 @@ class CompressionCache:
avg_ns = self._total_lookup_ns / self._lookup_count if self._lookup_count else 0
return {
"cache_hits": self._hits,
+ "cache_disk_hits": self._disk_hits,
"cache_skip_hits": self._skip_hits,
"cache_misses": self._misses,
"cache_evictions": self._evictions,
"cache_size": len(self._results),
+ "cache_bytes": self._bytes,
+ "cache_max_bytes": self._max_bytes,
"cache_skip_size": len(self._skip),
"cache_avg_lookup_ns": avg_ns,
}
@@ -1683,17 +1905,24 @@ class CompressionCache:
def register_on_clear(self, callback: Any) -> None:
"""Register a callback fired (outside the lock) on every ``clear()``.
- Lets state keyed by the same content hashes — e.g. the router's
+ Lets state keyed by the same content keys — e.g. the router's
``_frozen_verdicts`` store — be reset together with the cache so it
cannot outlive the results/skip tiers it shadows.
"""
self._on_clear.append(callback)
def clear(self) -> None:
- """Clear all entries (e.g., on session end). Thread-safe."""
+ """Clear all entries, including this router's rows on disk (other
+ workers with the same config share them). Thread-safe."""
with self._lock:
self._results.clear()
+ self._bytes = 0
self._skip.clear()
+ self._disk_reads.clear()
+ # Under the lock, so no disk read can start before the rows are
+ # gone and bring a cleared result back into memory.
+ if self._store is not None:
+ self._store.clear_prefix(self._namespace)
# Fire callbacks outside the lock to avoid cross-lock ordering issues.
for callback in self._on_clear:
callback()
@@ -1945,6 +2174,12 @@ class ContentRouterConfig:
# emits a `<<ccr:…>>` / `Retrieve …` retrieval marker. SmartCrusher is
# additionally forced marker-free via smart_crusher_lossless_only.
lossless: bool = False
+
+ # Back the compression cache with a SQLite file (see
+ # headroom.cache.result_store) so results its memory budget evicts, or a
+ # restart loses, come back without compressing again. The proxy turns it on;
+ # library callers keep the cache in memory unless they ask.
+ result_cache_disk: bool = False
# Cross-turn (whole-conversation) verbatim de-dup. Replaces a contiguous span
# in a later tool output that already appeared verbatim in an earlier tool
# output with an in-context pointer. Prefix-monotonic (cache-safe) and
@@ -2455,7 +2690,12 @@ class ContentRouter(Transform):
# are not gated. Now backed by the #3486 ContextVar state — see
# the ``_runtime_compression_policy`` property below.
- self._cache = CompressionCache()
+ store = None
+ if self.config.result_cache_disk:
+ from ..cache.result_store import open_result_store
+
+ store = open_result_store(ttl_seconds=RESULT_TTL_SECONDS)
+ self._cache = CompressionCache(store=store, namespace=_config_namespace(self.config))
# Cache-churn fix (HEADROOM_FREEZE_BLOCK_DECISION, default off):
# freeze each block's compress-vs-passthrough verdict on first
@@ -2473,7 +2713,7 @@ class ContentRouter(Transform):
# the parallel compression pass writes verdicts from worker threads, so
# we match that posture with a dedicated lock rather than relying on
# GIL atomicity (which would not protect the read-then-evict sequence).
- self._frozen_verdicts: dict[int, bool] = {}
+ self._frozen_verdicts: dict[str, bool] = {}
# The store is process-wide (one router per pipeline, shared by every
# session), so the cap must scale with the number of CONCURRENT
# sessions, not one user's workload: at org scale (many users behind
@@ -2636,7 +2876,7 @@ class ContentRouter(Transform):
f"preserved_chars~={preserved} (cache bust avoided)"
)
- def _record_frozen_verdict(self, content_key: int, verdict: bool) -> None:
+ def _record_frozen_verdict(self, content_key: str, verdict: bool) -> None:
"""Record a frozen verdict (thread-safe, bounded FIFO eviction).
Caps the store at ``_frozen_verdicts_max`` entries, evicting the
@@ -2653,7 +2893,7 @@ class ContentRouter(Transform):
del self._frozen_verdicts[oldest]
self._frozen_verdicts[content_key] = verdict
- def _get_frozen_verdict(self, content_key: int) -> bool | None:
+ def _get_frozen_verdict(self, content_key: str) -> bool | None:
"""Read a frozen verdict (thread-safe). Returns None if absent."""
with self._frozen_lock:
return self._frozen_verdicts.get(content_key)
@@ -6150,7 +6390,7 @@ class ContentRouter(Transform):
# enforce_reversibility, precomputed_detection). The detection is the one
# already computed in the routing loop below, threaded through so a
# cache-miss message is not re-detected inside compress().
- _PendingTask = tuple[int, str, str, float, int, bool, "DetectionResult | None"]
+ _PendingTask = tuple[int, str, str, float, str, bool, "DetectionResult | None"]
pending_tasks: list[_PendingTask] = []
# #856 P2b (flag-gated, default off): net-cost frozen-floor unlock.
@@ -6490,13 +6730,13 @@ class ContentRouter(Transform):
# Tier 2 (result): known compresses → reuse compressed text.
# Key on the runtime target_ratio too: the same content compressed at
# a different ratio is a different result, so it must not alias.
- content_key = hash((content, getattr(self, "_runtime_target_ratio", None)))
# Tool ground truth is gated against lossy-unrecoverable results below
# (#1307). Partition its cache namespace so a gated tool entry is never
# served from — or poisons — an ungated entry for byte-identical content.
enforce_reversibility = role == "tool"
- if enforce_reversibility:
- content_key = hash((content_key, True))
+ content_key = content_cache_key(
+ content, getattr(self, "_runtime_target_ratio", None), gated=enforce_reversibility
+ )
# Tier 1: skip set — instant rejection
if self._cache.is_skipped(content_key):
@@ -7560,7 +7800,9 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, was_compressed = self._compress_block_content(
content=tool_text,
- content_key=hash((tool_text, getattr(self, "_runtime_target_ratio", None))),
+ content_key=content_cache_key(
+ tool_text, getattr(self, "_runtime_target_ratio", None)
+ ),
context=block_context,
bias=bias,
min_ratio=min_ratio,
@@ -7630,8 +7872,8 @@ class ContentRouter(Transform):
# Two-tier compression cache → shared helper
compressed_content, _was_compressed = self._compress_block_content(
content=text_content,
- content_key=hash(
- (text_content, getattr(self, "_runtime_target_ratio", None))
+ content_key=content_cache_key(
+ text_content, getattr(self, "_runtime_target_ratio", None)
),
context=context,
bias=1.0,
@@ -7663,7 +7905,7 @@ class ContentRouter(Transform):
def _compress_block_content(
self,
content: str,
- content_key: int,
+ content_key: str,
context: str,
bias: float,
min_ratio: float,
@@ -7684,7 +7926,7 @@ class ContentRouter(Transform):
Args:
content: The string content to compress.
- content_key: Pre-computed ``hash(content)`` for cache lookups.
+ content_key: Pre-computed :func:`content_cache_key` for cache lookups.
context: User/query context for relevance-aware compression.
bias: Compression bias multiplier (tool-specific or 1.0).
min_ratio: Adaptive minimum compression ratio threshold.
diff --git a/tests/conftest.py b/tests/conftest.py
index 23d46bc65..33a8cd19e 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -80,7 +80,7 @@ def _skip_proxy_port_preflight_unless_exercised(
@pytest.fixture(autouse=True)
-def _scrub_developer_headroom_env(monkeypatch, tmp_path):
+def _scrub_developer_headroom_env(monkeypatch, tmp_path, tmp_path_factory):
for key in list(os.environ):
if key.startswith("HEADROOM_"):
monkeypatch.delenv(key, raising=False)
@@ -89,6 +89,11 @@ def _scrub_developer_headroom_env(monkeypatch, tmp_path):
# test its own store so proxy/CLI startup cannot load developer settings and
# saves cannot rewrite them. Tests of path precedence can override this.
monkeypatch.setenv("HEADROOM_SETTINGS_PATH", str(tmp_path / "headroom-settings.json"))
+ # The proxy's compression cache keeps results on disk; never in the
+ # developer's ~/.headroom, never shared between tests, and not in tmp_path,
+ # which tests assert stays empty when nothing is persisted.
+ cache_dir = tmp_path_factory.mktemp("compression-cache")
+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(cache_dir / "compression_cache.db"))
# The scrub also deletes the HEADROOM_HARD_WATCHDOG_SECS=0 opt-out CI sets
# (#3845), so every test that enters the proxy lifespan armed the 90s
# production watchdog inside pytest; it hard-exits the process with code 1
diff --git a/tests/test_router_result_cache.py b/tests/test_router_result_cache.py
new file mode 100644
index 000000000..2c2d8307a
--- /dev/null
+++ b/tests/test_router_result_cache.py
@@ -0,0 +1,382 @@
+"""The content router's compression cache stays within its memory budget."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import threading
+
+from headroom.cache.result_store import ResultStore, open_result_store
+from headroom.transforms.content_router import (
+ CompressionCache,
+ ContentRouter,
+ ContentRouterConfig,
+ content_cache_key,
+)
+
+
+def test_results_stay_within_the_memory_budget_least_recently_used_first():
+ cache = CompressionCache(max_bytes=10_000)
+ for i in range(100):
+ cache.put(f"k{i}", "x" * 1000, 0.5, "kompress")
+ cache.get("k0") # k0 stays recently used
+ assert cache.stats["cache_bytes"] <= 10_000
+ assert cache.get("k0") is not None
+ assert cache.get("k1") is None # evicted, not kept forever
+
+
+def test_an_evicted_result_comes_back_from_disk_unchanged(tmp_path):
+ cache = CompressionCache(max_bytes=3_000, store=ResultStore(tmp_path / "c.db", 1800))
+ cache.put("first", "compressed once", 0.4, "kompress")
+ for i in range(10):
+ cache.put(f"k{i}", "y" * 1000, 0.5, "kompress")
+ # Out of memory, but read back: no second compression, same bytes.
+ assert cache.get("first") == ("compressed once", 0.4, "kompress")
+ assert cache.stats["cache_disk_hits"] == 1
+
+
+def test_the_skip_set_is_bounded_too():
+ cache = CompressionCache(max_bytes=1024 * 1024)
+ for i in range(50_000):
+ cache.mark_skip(f"s{i}")
+ assert cache.skip_size <= 2000
+ assert cache.is_skipped("s49999")
+
+
+def test_keys_are_the_same_in_every_process():
+ code = (
+ "from headroom.transforms.content_router import content_cache_key;"
+ "print(content_cache_key('same text', 0.5, gated=True))"
+ )
+ other = subprocess.run(
+ [sys.executable, "-c", code], capture_output=True, text=True, check=True
+ ).stdout.strip()
+ assert other == content_cache_key("same text", 0.5, gated=True)
+ assert content_cache_key("same text", 0.5) != other # gated tool output kept apart
+
+
+def test_stateless_mode_writes_no_cache_file(tmp_path, monkeypatch):
+ monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(tmp_path))
+ monkeypatch.setenv("HEADROOM_STATELESS", "1")
+ assert open_result_store(1800) is None
+ assert not (tmp_path / "compression_cache.db").exists()
+
+
+def test_a_result_in_use_is_kept_and_an_idle_one_is_dropped(tmp_path, monkeypatch):
+ # Nothing signals a session's end: a result expires once unused for the
+ # ttl, from memory and from disk, while a session still reading it keeps it.
+ import time as time_module
+
+ clock = [1000.0]
+ monkeypatch.setattr(time_module, "monotonic", lambda: clock[0])
+ monkeypatch.setattr(time_module, "time", lambda: clock[0])
+ store = ResultStore(tmp_path / "c.db", ttl_seconds=100)
+ cache = CompressionCache(ttl_seconds=100, store=store)
+ cache.put("live", "kept", 0.5, "kompress")
+ cache.put("idle", "dropped", 0.5, "kompress")
+ for _ in range(5): # 'live' is read every 60 s, for 300 s
+ clock[0] += 60
+ assert cache.get("live") is not None
+ assert cache.get("idle") is None
+ assert store.get(cache._namespace + "idle") is None
+ assert store.get(cache._namespace + "live") == ("kept", 0.5, "kompress")
+
+
+def test_routers_configured_differently_never_share_output(tmp_path, monkeypatch):
+ # The proxy can run two routers (per-provider Kompress overrides), and a
+ # restart can change settings; both read the same file.
+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "c.db"))
+ lossy = ContentRouter(ContentRouterConfig(result_cache_disk=True))
+ lossless = ContentRouter(ContentRouterConfig(result_cache_disk=True, lossless=True))
+ lossy._cache.put("k", "lossy output", 0.3, "kompress")
+ assert lossless._cache.get("k") is None
+ same = ContentRouter(ContentRouterConfig(result_cache_disk=True))
+ assert same._cache.get("k") == ("lossy output", 0.3, "kompress")
+
+
+def test_the_namespace_is_the_same_in_every_process():
+ code = (
+ "from headroom.transforms.content_router import ContentRouterConfig, _config_namespace;"
+ "print(_config_namespace(ContentRouterConfig()))"
+ )
+ seen = {
+ subprocess.run(
+ [sys.executable, "-c", code],
+ capture_output=True,
+ text=True,
+ check=True,
+ env={**__import__("os").environ, "PYTHONHASHSEED": seed},
+ ).stdout.strip()
+ for seed in ("1", "2")
+ }
+ assert len(seen) == 1
+
+
+def test_a_broken_database_is_a_miss_not_a_failed_request(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(store=store)
+ store._db.close()
+ cache.put("k", "text", 0.5, "kompress") # no raise
+ cache.clear()
+ assert cache.get("k") is None
+
+
+def test_a_result_pointing_at_a_ccr_original_is_not_kept_on_disk(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(store=store)
+ cache.put("k", "first lines <<ccr:0123456789abcdef>>", 0.2, "x")
+ assert store.get(cache._namespace + "k") is None
+
+
+def test_a_nonfinite_budget_falls_back_instead_of_stopping_startup(monkeypatch):
+ for raw in ("nan", "inf", "-5"):
+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_MB", raw)
+ assert CompressionCache().stats["cache_max_bytes"] == 64 * 1024 * 1024
+
+
+def test_a_changed_compressor_setting_does_not_reuse_old_results(tmp_path, monkeypatch):
+ # Settings the router reads from the environment, outside its config.
+ monkeypatch.setenv("HEADROOM_COMPRESSION_CACHE_PATH", str(tmp_path / "c.db"))
+ before = ContentRouter(ContentRouterConfig(result_cache_disk=True))
+ before._cache.put("k", "made under the old setting", 0.3, "kompress")
+ monkeypatch.setenv("HEADROOM_LOSSLESS_THEN_LOSSY", "1")
+ after = ContentRouter(ContentRouterConfig(result_cache_disk=True))
+ assert after._cache.get("k") is None
+
+
+def test_a_marker_result_replacing_a_plain_one_removes_it_from_disk(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=4_000, store=store)
+ cache.put("k", "plain result", 0.5, "kompress")
+ cache.put("k", "first lines <<ccr:0123456789abcdef>>", 0.2, "x")
+ for i in range(10): # push "k" out of memory
+ cache.put(f"filler{i}", "f" * 1000, 0.5, "kompress")
+ assert cache.get("k") is None
+
+
+def test_clear_also_clears_the_routers_rows_on_disk(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(store=store, namespace="ns1:")
+ other = CompressionCache(store=store, namespace="ns2:")
+ cache.put("k", "value", 0.5, "kompress")
+ other.put("k", "theirs", 0.5, "kompress")
+ cache.clear()
+ assert cache.get("k") is None
+ assert store.get("ns2:k") == ("theirs", 0.5, "kompress")
+
+
+def test_an_older_put_never_undoes_a_newer_ones_disk_row(tmp_path):
+ # A marker put admitted first but reaching the disk after a newer plain
+ # put must not delete the plain result's row.
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(store=store)
+
+ class NewerPutRunsFirst:
+ def __init__(self) -> None:
+ self._lock, self._pending = threading.Lock(), True
+
+ def __enter__(self):
+ if self._pending: # between the marker's admit and its disk step
+ self._pending = False
+ cache.put("k", "plain output", 0.5, "text")
+ self._lock.acquire()
+
+ def __exit__(self, *exc):
+ self._lock.release()
+
+ cache._disk_lock = NewerPutRunsFirst()
+ cache.put("k", "<<ccr:0123456789abcdef 3_rows_offloaded>>", 0.5, "text")
+ assert store.get(cache._namespace + "k")[0] == "plain output"
+
+
+def test_a_cache_without_a_namespace_clears_only_its_own_rows(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ mine = CompressionCache(store=store)
+ other = CompressionCache(store=store, namespace="router-b:")
+ mine.put("a", "result a", 0.5, "text")
+ other.put("b", "result b", 0.5, "text")
+ mine.clear()
+ assert store.get("router-b:b") is not None
+
+
+def test_a_disk_read_in_flight_never_replaces_a_newer_put(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=600, store=store)
+ cache.put("k", "old", 0.5, "text")
+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory
+ read_done, release = threading.Event(), threading.Event()
+ real_get = store.get
+
+ def paused_get(key):
+ found = real_get(key)
+ store.get = real_get # pause only the first read
+ read_done.set()
+ assert release.wait(5)
+ return found
+
+ store.get = paused_get
+ results = []
+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))
+ reader.start()
+ assert read_done.wait(5) # the read has "old" in hand
+ cache.put("k", "new", 0.5, "text")
+ release.set()
+ reader.join(5)
+ assert results == [("new", 0.5, "text")]
+ assert cache.get("k") == ("new", 0.5, "text")
+ assert store.get(cache._namespace + "k")[0] == "new"
+
+
+def test_a_disk_read_during_clear_never_restores_a_cleared_result(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=600, store=store)
+ cache.put("k", "old", 0.5, "text")
+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory
+ deleting, release = threading.Event(), threading.Event()
+ real_clear_prefix = store.clear_prefix
+
+ def paused_clear_prefix(prefix):
+ deleting.set()
+ assert release.wait(5)
+ real_clear_prefix(prefix)
+
+ store.clear_prefix = paused_clear_prefix
+ clearer = threading.Thread(target=cache.clear)
+ clearer.start()
+ assert deleting.wait(5) # memory cleared, disk row not yet deleted
+ reader = threading.Thread(target=cache.get, args=("k",))
+ reader.start()
+ reader.join(0.5) # finishes here unless clear() still holds it off
+ release.set()
+ clearer.join(5)
+ reader.join(5)
+ assert store.get(cache._namespace + "k") is None
+ assert cache.get("k") is None
+
+
+class PauseAfterFetch:
+ """Stands in for a cache's ``_disk_read_lock``: the first ``pauses`` reads
+ stop after their fetch, outside the lock and before admission."""
+
+ def __init__(self, pauses: int) -> None:
+ self._lock = threading.Lock()
+ self.fetched = [threading.Event() for _ in range(pauses)]
+ self.release = [threading.Event() for _ in range(pauses)]
+ self._exits = 0
+
+ def __enter__(self):
+ self._lock.acquire()
+
+ def __exit__(self, *exc):
+ i, self._exits = self._exits, self._exits + 1
+ self._lock.release()
+ if i < len(self.fetched):
+ self.fetched[i].set()
+ assert self.release[i].wait(5)
+
+
+def test_concurrent_disk_reads_of_one_key_each_return_the_result(tmp_path):
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=600, store=store)
+ cache.put("k", "kept", 0.5, "text")
+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory
+ reads = cache._disk_read_lock = PauseAfterFetch(2)
+ results: dict[str, object] = {}
+ first = threading.Thread(target=lambda: results.update(first=cache.get("k")))
+ second = threading.Thread(target=lambda: results.update(second=cache.get("k")))
+ first.start()
+ assert reads.fetched[0].wait(5)
+ second.start()
+ assert reads.fetched[1].wait(5) # both reads in flight; the first finishes first
+ reads.release[0].set()
+ first.join(5)
+ reads.release[1].set()
+ second.join(5)
+ assert results == {"first": ("kept", 0.5, "text"), "second": ("kept", 0.5, "text")}
+
+
+def test_an_older_disk_read_never_replaces_a_later_reads_result(tmp_path):
+ # Two workers share one store: a read that fetched earlier holds an older
+ # row than one that fetched after the other worker's put.
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=600, store=store)
+ other_worker = CompressionCache(store=store)
+ cache.put("k", "old", 0.5, "text")
+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory
+ reads = cache._disk_read_lock = PauseAfterFetch(1)
+ results = []
+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))
+ reader.start()
+ assert reads.fetched[0].wait(5) # the first read has "old" in hand
+ other_worker.put("k", "new", 0.5, "text")
+ assert cache.get("k") == ("new", 0.5, "text") # a later read admits "new"
+ reads.release[0].set()
+ reader.join(5)
+ assert results == [("new", 0.5, "text")]
+ assert cache.get("k") == ("new", 0.5, "text")
+
+
+def test_a_read_that_fetches_later_is_never_outranked_by_an_older_row(tmp_path):
+ # Read A is numbered before read B but fetches after another worker's
+ # put: what A fetched is the newest row and must be what memory keeps.
+ store = ResultStore(tmp_path / "c.db", 1800)
+ cache = CompressionCache(max_bytes=600, store=store)
+ other_worker = CompressionCache(store=store)
+ cache.put("k", "old", 0.5, "text")
+ cache.put("other", "z" * 300, 0.5, "text") # evicts k from memory
+ fetching, release = threading.Event(), threading.Event()
+ real_get = store.get
+
+ def paused_get(key):
+ store.get = real_get # pause only the first read, before it fetches
+ fetching.set()
+ assert release.wait(5)
+ return real_get(key)
+
+ store.get = paused_get
+ results = []
+ first = threading.Thread(target=lambda: results.append(cache.get("k")))
+ first.start()
+ assert fetching.wait(5) # A is numbered; its fetch has not run
+ second = threading.Thread(target=cache.get, args=("k",))
+ second.start()
+ second.join(0.5) # B reads "old" here unless A's read holds it off
+ other_worker.put("k", "new", 0.5, "text")
+ release.set()
+ first.join(5)
+ second.join(5)
+ assert results == [("new", 0.5, "text")]
+ assert cache.get("k") == ("new", 0.5, "text")
+ assert store.get(cache._namespace + "k")[0] == "new"
+
+
+def test_a_result_that_expires_while_a_read_waits_is_not_returned(tmp_path, monkeypatch):
+ # get() misses memory, then waits for the disk-read lock; a marker result
+ # admitted meanwhile expires before the wait ends. Its second look at
+ # memory must apply the same lifetime as the first, not return it.
+ import time as time_module
+
+ clock = [1000.0]
+ monkeypatch.setattr(time_module, "monotonic", lambda: clock[0])
+ cache = CompressionCache(store=ResultStore(tmp_path / "c.db", 1800), marker_ttl_seconds=1)
+ waiting, real = threading.Event(), cache._disk_read_lock
+
+ class SignalsWait:
+ def __enter__(self):
+ waiting.set()
+ return real.__enter__()
+
+ def __exit__(self, *exc):
+ return real.__exit__(*exc)
+
+ cache._disk_read_lock = SignalsWait()
+ results = []
+ with real:
+ reader = threading.Thread(target=lambda: results.append(cache.get("k")))
+ reader.start()
+ assert waiting.wait(5)
+ cache.put("k", "head <<ccr:0123456789abcdef>>", 0.2, "x")
+ clock[0] += 2
+ reader.join(5)
+ assert results == [None]
+ assert cache.get("k") is None