Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
perf(cache): drop eager full-backend TTL sweep on every new-key store() (#3256)
Omar Gerardo2fed685e7773
diff --git a/headroom/cache/backends/sqlite.py b/headroom/cache/backends/sqlite.pyindex fe4233d41..ab5a35bed 100644--- a/headroom/cache/backends/sqlite.py+++ b/headroom/cache/backends/sqlite.py@@ -252,6 +252,16 @@ class SQLiteBackend:return 0return int(row[0])+ def external_revision(self) -> int | None:+ """Return a token that changes when another connection commits."""+ with self._lock:+ try:+ row = self._conn.execute("PRAGMA data_version").fetchone()+ except sqlite3.DatabaseError as e:+ self._handle_db_error(e, "revision")+ return None+ return int(row[0])+def keys(self) -> list[str]:with self._lock:try:diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.pyindex c5914772c..1712fa007 100644--- a/headroom/cache/compression_store.py+++ b/headroom/cache/compression_store.py@@ -268,10 +268,16 @@ class CompressionStore:# MEDIUM FIX #16: Use a min-heap for O(log n) eviction instead of O(n)# Heap entries are (created_at, hash_key) tuplesself._eviction_heap: list[tuple[float, str]] = []+ # Expiration ordering is separate so live eviction stays oldest-first.+ self._expiration_heap: list[tuple[float, float, str]] = []# CRITICAL FIX: Track stale entries count to know when heap cleanup is neededself._stale_heap_entries = 0+ self._stale_expiration_heap_entries = 0# Threshold for triggering heap rebuild (when 50% are stale)self._heap_rebuild_threshold = 0.5+ external_revision = getattr(self._backend, "external_revision", None)+ self._backend_revision_reader = external_revision if callable(external_revision) else None+ self._backend_revision = self._read_backend_revision()@propertydef default_ttl_seconds(self) -> int:@@ -398,6 +404,7 @@ class CompressionStore:tool_signature_hash=tool_signature_hash,compression_strategy=compression_strategy,)+ expires_at = self._expiration_time(entry)# Process pending feedback BEFORE acquiring lock for eviction.# This ensures feedback from entries about to be evicted is captured.@@ -434,11 +441,17 @@ class CompressionStore:hash_key,)# Mark old heap entry as stale since we're replacing it.- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()self._backend.set(hash_key, entry)# MEDIUM FIX #16: Add to eviction heap for O(log n) evictionheapq.heappush(self._eviction_heap, (entry.created_at, hash_key))+ heapq.heappush(+ self._expiration_heap,+ (expires_at, entry.created_at, hash_key),+ )+ if existing is not None:+ self._rebuild_heap_if_needed()return hash_key@@ -465,7 +478,7 @@ class CompressionStore:if entry.is_expired():self._backend.delete(hash_key)# CRITICAL FIX: Track stale heap entry- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()return None# Track access for feedback@@ -528,7 +541,7 @@ class CompressionStore:if entry.is_expired():self._backend.delete(hash_key)- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()return Nonereturn {@@ -597,7 +610,7 @@ class CompressionStore:if clean_expired:self._backend.delete(hash_key)# CRITICAL FIX: Track stale heap entry- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()return Falsereturn True@@ -633,7 +646,7 @@ class CompressionStore:if expired and clean_expired:self._backend.delete(hash_key)- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()return status@@ -685,6 +698,7 @@ class CompressionStore:# Add eviction heap memorybytes_used += sys.getsizeof(self._eviction_heap)+ bytes_used += sys.getsizeof(self._expiration_heap)return ComponentStats(name="compression_store",@@ -728,24 +742,66 @@ class CompressionStore:self._retrieval_events.clear()self._pending_feedback_events.clear()self._eviction_heap.clear() # MEDIUM FIX #16: Clear heap too+ self._expiration_heap.clear()self._stale_heap_entries = 0 # CRITICAL FIX: Reset stale counter+ self._stale_expiration_heap_entries = 0def _evict_if_needed(self) -> None:- """Evict old entries if at capacity. Must be called with lock held.+ """Evict expired, then oldest live entries until below capacity.MEDIUM FIX #16: Use heap for O(log n) eviction instead of O(n) scan.CRITICAL FIX: Track and clean stale heap entries to prevent memory leak."""- # First, remove expired entries- self._clean_expired()-- # CRITICAL FIX: Rebuild heap if too many stale entries- # This prevents unbounded heap growth when entries are deleted/replaced- heap_size = len(self._eviction_heap)- if heap_size > 0:- stale_ratio = self._stale_heap_entries / heap_size- if stale_ratio >= self._heap_rebuild_threshold:+ # Avoid the old full backend items() scan per new-key store().+ # The expiration heap keeps cleanup proportional to expired roots.++ backend_count = self._backend.count()+ indexed_count = max(0, len(self._eviction_heap) - self._stale_heap_entries)+ at_capacity = backend_count >= self._max_entries+ if at_capacity:+ backend_revision = self._read_backend_revision()+ backend_changed = self._backend_revision_reader is not None and (+ backend_revision is None or backend_revision != self._backend_revision+ )+ if indexed_count != backend_count or backend_changed:self._rebuild_heap()+ else:+ self._rebuild_heap_if_needed()+ if backend_revision is not None:+ self._backend_revision = backend_revision+ else:+ self._rebuild_heap_if_needed()++ # Remove expired entries first without reporting successful compression.+ while self._expiration_heap:+ expires_at, created_at, hash_key = self._expiration_heap[0]+ if expires_at >= time.time():+ break++ entry = self._backend.get(hash_key)+ if entry is None and at_capacity:+ entry = self._backend.get(hash_key)+ if entry is None:+ return+ heapq.heappop(self._expiration_heap)+ if (+ entry is not None+ and entry.created_at == created_at+ and self._expiration_time(entry) == expires_at+ ):+ self._backend.delete(hash_key)+ self._stale_heap_entries += 1+ elif self._stale_expiration_heap_entries > 0:+ self._stale_expiration_heap_entries -= 1+ else:+ # Backend-side TTL purges leave the matching eviction tuple stale.+ self._stale_heap_entries += 1++ if not at_capacity or self._backend.count() < self._max_entries:+ return++ if not at_capacity:+ return# If still at capacity, remove oldest entries using heapwhile self._backend.count() >= self._max_entries and self._eviction_heap:@@ -756,19 +812,56 @@ class CompressionStore:# (entry might have been deleted or replaced)entry = self._backend.get(hash_key)if entry is not None and entry.created_at == created_at:- # HIGH FIX: Track eviction as "successful compression" if never retrieved- # This prevents state divergence between store and feedback loop- if self._enable_feedback and entry.retrieval_count == 0:- # Entry was never retrieved = compression was successful- # Notify feedback system so it knows this strategy worked+ if not entry.is_expired() and self._enable_feedback and entry.retrieval_count == 0:self._record_eviction_success(entry)self._backend.delete(hash_key)+ self._stale_expiration_heap_entries += 1else:# CRITICAL FIX: This was a stale entry, decrement counter# (we already popped it, so the stale entry is now gone)if self._stale_heap_entries > 0:self._stale_heap_entries -= 1+ def _mark_heap_entries_stale(self) -> None:+ """Record lazy tuples invalidated by deleting or replacing an entry."""+ self._stale_heap_entries += 1+ self._stale_expiration_heap_entries += 1++ @staticmethod+ def _expiration_time(entry: CompressionEntry) -> float:+ """Return the entry expiration time, saturating numeric overflow."""+ try:+ return entry.created_at + float(entry.ttl)+ except OverflowError:+ return float("inf") if entry.ttl > 0 else float("-inf")++ def _read_backend_revision(self) -> object | None:+ """Read optional backend revision metadata without breaking stores."""+ if self._backend_revision_reader is None:+ return None+ try:+ revision: object | None = self._backend_revision_reader()+ return revision+ except Exception:+ logger.debug("CCR backend revision lookup failed", exc_info=True)+ return None++ def _rebuild_heap_if_needed(self) -> None:+ """Rebuild both indexes when either stale ratio reaches the threshold."""+ heap_size = len(self._eviction_heap)+ expiration_heap_size = len(self._expiration_heap)+ stale_ratio = self._stale_heap_entries / heap_size if heap_size else 0+ stale_expiration_ratio = (+ self._stale_expiration_heap_entries / expiration_heap_size+ if expiration_heap_size+ else 0+ )+ if (+ stale_ratio >= self._heap_rebuild_threshold+ or stale_expiration_ratio >= self._heap_rebuild_threshold+ ):+ self._rebuild_heap()+def _clean_expired(self) -> None:"""Remove expired entries. Must be called with lock held.@@ -782,9 +875,7 @@ class CompressionStore:expired_keys = [key for key, entry in self._backend.items() if entry.is_expired()]for key in expired_keys:self._backend.delete(key)- # CRITICAL FIX: Increment stale counter - the heap still has an entry- # for this key that will be stale when we try to evict- self._stale_heap_entries += 1+ self._mark_heap_entries_stale()def _rebuild_heap(self) -> None:"""Rebuild heap from current store entries. Must be called with lock held.@@ -792,15 +883,20 @@ class CompressionStore:CRITICAL FIX: This removes stale heap entries that accumulate when entriesare deleted or replaced. Without this, the heap grows unboundedly."""- # Build new heap from current store entries only- self._eviction_heap = [- (entry.created_at, hash_key) for hash_key, entry in self._backend.items()+ # Reuse one snapshot so rebuilding never doubles backend enumeration.+ entries = self._backend.items()+ self._eviction_heap = [(entry.created_at, hash_key) for hash_key, entry in entries]+ self._expiration_heap = [+ (self._expiration_time(entry), entry.created_at, hash_key)+ for hash_key, entry in entries]heapq.heapify(self._eviction_heap)+ heapq.heapify(self._expiration_heap)# Reset stale counter - heap is now cleanself._stale_heap_entries = 0+ self._stale_expiration_heap_entries = 0logger.debug(- "Rebuilt eviction heap: %d entries",+ "Rebuilt eviction heaps: %d entries",len(self._eviction_heap),)diff --git a/tests/test_compression_store.py b/tests/test_compression_store.pyindex 6d9bfdc2b..9f709d3b6 100644--- a/tests/test_compression_store.py+++ b/tests/test_compression_store.py@@ -26,6 +26,8 @@ from unittest.mock import MagicMock, patchimport pytest+from headroom.cache.backends.memory import InMemoryBackend+from headroom.cache.backends.sqlite import SQLiteBackendfrom headroom.cache.compression_store import (CCR_TTL_SECONDS_ENV,DEFAULT_CCR_TTL_SECONDS,@@ -662,6 +664,126 @@ class TestCompressionStoreTTL:class TestCompressionStoreEviction:"""Tests for CompressionStore memory limits and eviction."""+ def test_new_key_store_does_not_scan_backend_for_expiry(self):+ """New-key store() must not trigger a backend items() expiry scan."""++ class _ItemsCountingBackend(InMemoryBackend):+ def __init__(self) -> None:+ super().__init__()+ self.items_calls = 0++ def items(self) -> list[tuple[str, CompressionEntry]]:+ self.items_calls += 1+ return super().items()++ backend = _ItemsCountingBackend()+ store = CompressionStore(max_entries=2, backend=backend)++ for i in range(3):+ store.store(original=f"content_{i}", compressed=f"compressed_{i}")++ assert backend.items_calls == 0++ def test_expired_cleanup_below_capacity_is_bounded(self):+ """A new key cleans at most one expired entry while capacity is available."""+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=10, enable_feedback=False, backend=backend)++ with (+ patch("headroom.cache.compression_store.time.time") as now,+ patch.object(backend, "get", wraps=backend.get) as get,+ patch.object(backend, "delete", wraps=backend.delete) as delete,+ patch.object(backend, "items", wraps=backend.items) as items,+ patch.object(backend, "count", wraps=backend.count) as count,+ ):+ operations = (get, delete, items, count)+ now.return_value = 100+ for i in range(4):+ store.store(original=f"expired_{i}", compressed="expired", ttl=1)++ for operation in operations:+ operation.reset_mock()+ now.return_value = 200+ store.store(original="new_1", compressed="new", ttl=100)+ assert tuple(operation.call_count for operation in operations) == (2, 1, 0, 1)++ for operation in operations:+ operation.reset_mock()+ now.return_value = 201+ store.store(original="new_2", compressed="new", ttl=100)+ assert (items.call_count, count.call_count) == (0, 1)++ def test_in_memory_oversized_ttl_at_capacity_keeps_indexes_consistent(self):+ """An oversized positive TTL is stored as a non-expiring heap entry."""+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=1, enable_feedback=False, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 100.0+ old_hash = store.store(original="old", compressed="old")+ now.return_value = 101.0+ oversized_hash = store.store(+ original="oversized",+ compressed="oversized",+ ttl=10**1000,+ )++ entry = backend.get(oversized_hash)+ assert entry is not None+ assert not backend.exists(old_hash)+ assert len(store._eviction_heap) - store._stale_heap_entries == backend.count() == 1+ assert len(store._expiration_heap) - store._stale_expiration_heap_entries == backend.count()+ assert (entry.created_at, oversized_hash) in store._eviction_heap+ assert (float("inf"), entry.created_at, oversized_hash) in store._expiration_heap++ @pytest.mark.parametrize("mode", ["count", "revision", "revision_error", "stale_revision"])+ def test_shared_sqlite_pressure_reconciles_sibling_entries(self, tmp_path, mode: str):+ """Capacity reconciliation covers sibling count and identity changes."""++ class _RevisionFailureSQLiteBackend(SQLiteBackend):+ fail_revision = False++ def external_revision(self) -> int | None:+ if self.fail_revision:+ raise RuntimeError("revision unavailable")+ return super().external_revision()++ db_path = tmp_path / "ccr.db"+ backend = _RevisionFailureSQLiteBackend(db_path)+ sibling_backend = SQLiteBackend(db_path)+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)+ sibling_store = CompressionStore(enable_feedback=False, backend=sibling_backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 90+ removed_hash = None+ if mode != "count":+ removed_hash = store.store(+ original="removed",+ compressed="removed",+ ttl=1 if mode == "stale_revision" else 100,+ )+ now.return_value = 100+ live_hash = store.store(+ original="live",+ compressed="live",+ ttl=100,+ tool_signature_hash="live_signature",+ compression_strategy="top_k",+ )+ if removed_hash is not None and mode != "stale_revision":+ sibling_backend.delete(removed_hash)+ now.return_value = 110+ sibling_store.store(original="expired", compressed="expired", ttl=5)+ backend.fail_revision = mode == "revision_error"+ now.return_value = 116+ with patch.object(backend, "items", wraps=backend.items) as items:+ new_hash = store.store(original="new", compressed="new", ttl=100)++ assert set(backend.keys()) == {live_hash, new_hash}+ assert items.call_count == 1+ assert store._pending_feedback_events == []+def test_eviction_at_capacity(self, store_with_small_capacity: CompressionStore):"""Oldest entries are evicted when at capacity."""hashes = []@@ -708,6 +830,24 @@ class TestCompressionStoreEviction:assert store_with_small_capacity.exists(hashes[2])assert store_with_small_capacity.exists(new_hash)+ def test_mixed_ttl_pressure_evicts_oldest_created_live_entry(self):+ """Live eviction remains creation-ordered across mixed TTLs."""+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 100+ older_long_ttl = store.store(original="older", compressed="older", ttl=100)+ now.return_value = 110+ newer_short_ttl = store.store(original="newer", compressed="newer", ttl=20)+ now.return_value = 120+ new_hash = store.store(original="new", compressed="new", ttl=100)++ assert not backend.exists(older_long_ttl)+ assert backend.exists(newer_short_ttl)+ assert backend.exists(new_hash)+ assert backend.count() == 2+def test_duplicate_store_at_capacity_does_not_evict(self, store_with_small_capacity: CompressionStore):@@ -736,25 +876,109 @@ class TestCompressionStoreEviction:def test_eviction_cleans_expired_first(self):"""Eviction cleans expired entries before evicting valid ones."""- store = CompressionStore(max_entries=3, default_ttl=1)-- # Add 2 entries that will expire- hash1 = store.store(original="content_1", compressed="c1", ttl=1)- hash2 = store.store(original="content_2", compressed="c2", ttl=1)-- time.sleep(1.1) # Wait for expiration-- # Add 2 more entries (should clean expired first, not evict new)- hash3 = store.store(original="content_3", compressed="c3", ttl=300)- hash4 = store.store(original="content_4", compressed="c4", ttl=300)-- # Expired entries should be gone- assert not store.exists(hash1)- assert not store.exists(hash2)-- # New entries should exist- assert store.exists(hash3)- assert store.exists(hash4)+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 100+ live_hash = store.store(original="live", compressed="live", ttl=100)+ now.return_value = 110+ expired_hash = store.store(+ original="expired",+ compressed="expired",+ ttl=5,+ tool_signature_hash="expired_signature",+ compression_strategy="top_k",+ )+ now.return_value = 116+ new_hash = store.store(original="new", compressed="new", ttl=100)++ assert backend.exists(live_hash)+ assert not backend.exists(expired_hash)+ assert backend.exists(new_hash)+ assert backend.count() == 2+ assert store._pending_feedback_events == []++ @pytest.mark.parametrize("misses", [1, 2])+ def test_expiration_cleanup_handles_transient_backend_misses(self, misses: int):+ """Expiry lookup misses retry once, then fail open without live eviction."""++ class _TransientMissBackend(InMemoryBackend):+ miss_key: str | None = None+ misses_remaining = 0++ def get(self, hash_key: str) -> CompressionEntry | None:+ if hash_key == self.miss_key and self.misses_remaining:+ self.misses_remaining -= 1+ return None+ return super().get(hash_key)++ backend = _TransientMissBackend()+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ if misses == 1:+ now.return_value = 80+ store.store(original="stale", compressed="stale", ttl=100)+ now.return_value = 90+ live_hash = store.store(+ original="live",+ compressed="live",+ ttl=100,+ tool_signature_hash="live_signature",+ compression_strategy="top_k",+ )+ now.return_value = 91+ expired_hash = store.store(original="expired", compressed="expired", ttl=5)+ backend.miss_key = expired_hash+ backend.misses_remaining = misses+ now.return_value = 97+ new_hash = store.store(original="new", compressed="new", ttl=100)++ expected_keys = {live_hash, new_hash}+ if misses == 2:+ expected_keys.add(expired_hash)+ assert set(backend.keys()) == expected_keys+ assert store._pending_feedback_events == []++ def test_eviction_ignores_stale_expiration_after_replacement(self):+ """A replaced entry survives its old expiration heap tuple."""+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 90+ oldest_hash = store.store(original="oldest", compressed="oldest", ttl=100)+ now.return_value = 100+ replacement_hash = store.store(original="replacement", compressed="v1", ttl=5)+ now.return_value = 101+ store.store(original="replacement", compressed="v2", ttl=100)+ now.return_value = 110+ new_hash = store.store(original="new", compressed="new", ttl=100)++ assert not backend.exists(oldest_hash)+ assert backend.exists(replacement_hash)+ assert backend.exists(new_hash)++ def test_eviction_ignores_stale_indexes_after_read_cleanup(self):+ """Read-side expiry cleanup leaves both eviction indexes consistent."""+ backend = InMemoryBackend()+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)++ with patch("headroom.cache.compression_store.time.time") as now:+ now.return_value = 100+ expired_hash = store.store(original="expired", compressed="expired", ttl=1)+ now.return_value = 102+ assert store.retrieve(expired_hash) is None+ first_live = store.store(original="first", compressed="first", ttl=100)+ now.return_value = 103+ second_live = store.store(original="second", compressed="second", ttl=100)+ now.return_value = 104+ new_hash = store.store(original="new", compressed="new", ttl=100)++ assert not backend.exists(first_live)+ assert backend.exists(second_live)+ assert backend.exists(new_hash)def test_heap_rebuild_on_stale_threshold(self):"""Heap is rebuilt when stale entry ratio exceeds threshold."""@@ -768,8 +992,10 @@ class TestCompressionStoreEviction:for i in range(5):store.store(original=f"content_{i}", compressed=f"updated_{i}")- # Stale ratio should be tracked- # The heap rebuild happens automatically when threshold is exceeded+ assert len(store._eviction_heap) == 5+ assert len(store._expiration_heap) == 5+ assert store._stale_heap_entries == 0+ assert store._stale_expiration_heap_entries == 0# =============================================================================@@ -1054,6 +1280,10 @@ class TestCompressionStoreEdgeCases:stats = store.get_stats()assert stats["entry_count"] == 0+ assert store._eviction_heap == []+ assert store._expiration_heap == []+ assert store._stale_heap_entries == 0+ assert store._stale_expiration_heap_entries == 0def test_clear_removes_retrieval_events(self, store: CompressionStore):"""clear() removes retrieval events."""@@ -1263,7 +1493,7 @@ class TestCompressionStoreFeedback:store = CompressionStore(max_entries=2, enable_feedback=True)# Store entries with signature hash for eviction tracking- store.store(+ first_hash = store.store(original="content_0",compressed="c0",tool_signature_hash="sig_0",@@ -1290,6 +1520,9 @@ class TestCompressionStoreFeedback:# The evicted entry (content_0) was never retrieved,# so an eviction_success event should be queued# (tested via the pending_feedback mechanism)+ assert len(store._pending_feedback_events) == 1+ assert store._pending_feedback_events[0].hash == first_hash+ assert store._pending_feedback_events[0].retrieval_type == "eviction_success"# =============================================================================