Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

perf(cache): drop eager full-backend TTL sweep on every new-key store() (#3256)

Omar Gerardo2fed685e7773
diff --git a/headroom/cache/backends/sqlite.py b/headroom/cache/backends/sqlite.py
index fe4233d41..ab5a35bed 100644
--- a/headroom/cache/backends/sqlite.py
+++ b/headroom/cache/backends/sqlite.py
@@ -252,6 +252,16 @@ class SQLiteBackend:
return 0
return int(row[0])
+ def external_revision(self) -> int | None:
+ """Return a token that changes when another connection commits."""
+ with self._lock:
+ try:
+ row = self._conn.execute("PRAGMA data_version").fetchone()
+ except sqlite3.DatabaseError as e:
+ self._handle_db_error(e, "revision")
+ return None
+ return int(row[0])
+
def keys(self) -> list[str]:
with self._lock:
try:
diff --git a/headroom/cache/compression_store.py b/headroom/cache/compression_store.py
index c5914772c..1712fa007 100644
--- a/headroom/cache/compression_store.py
+++ b/headroom/cache/compression_store.py
@@ -268,10 +268,16 @@ class CompressionStore:
# MEDIUM FIX #16: Use a min-heap for O(log n) eviction instead of O(n)
# Heap entries are (created_at, hash_key) tuples
self._eviction_heap: list[tuple[float, str]] = []
+ # Expiration ordering is separate so live eviction stays oldest-first.
+ self._expiration_heap: list[tuple[float, float, str]] = []
# CRITICAL FIX: Track stale entries count to know when heap cleanup is needed
self._stale_heap_entries = 0
+ self._stale_expiration_heap_entries = 0
# Threshold for triggering heap rebuild (when 50% are stale)
self._heap_rebuild_threshold = 0.5
+ external_revision = getattr(self._backend, "external_revision", None)
+ self._backend_revision_reader = external_revision if callable(external_revision) else None
+ self._backend_revision = self._read_backend_revision()
@property
def default_ttl_seconds(self) -> int:
@@ -398,6 +404,7 @@ class CompressionStore:
tool_signature_hash=tool_signature_hash,
compression_strategy=compression_strategy,
)
+ expires_at = self._expiration_time(entry)
# Process pending feedback BEFORE acquiring lock for eviction.
# This ensures feedback from entries about to be evicted is captured.
@@ -434,11 +441,17 @@ class CompressionStore:
hash_key,
)
# Mark old heap entry as stale since we're replacing it.
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
self._backend.set(hash_key, entry)
# MEDIUM FIX #16: Add to eviction heap for O(log n) eviction
heapq.heappush(self._eviction_heap, (entry.created_at, hash_key))
+ heapq.heappush(
+ self._expiration_heap,
+ (expires_at, entry.created_at, hash_key),
+ )
+ if existing is not None:
+ self._rebuild_heap_if_needed()
return hash_key
@@ -465,7 +478,7 @@ class CompressionStore:
if entry.is_expired():
self._backend.delete(hash_key)
# CRITICAL FIX: Track stale heap entry
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
return None
# Track access for feedback
@@ -528,7 +541,7 @@ class CompressionStore:
if entry.is_expired():
self._backend.delete(hash_key)
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
return None
return {
@@ -597,7 +610,7 @@ class CompressionStore:
if clean_expired:
self._backend.delete(hash_key)
# CRITICAL FIX: Track stale heap entry
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
return False
return True
@@ -633,7 +646,7 @@ class CompressionStore:
if expired and clean_expired:
self._backend.delete(hash_key)
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
return status
@@ -685,6 +698,7 @@ class CompressionStore:
# Add eviction heap memory
bytes_used += sys.getsizeof(self._eviction_heap)
+ bytes_used += sys.getsizeof(self._expiration_heap)
return ComponentStats(
name="compression_store",
@@ -728,24 +742,66 @@ class CompressionStore:
self._retrieval_events.clear()
self._pending_feedback_events.clear()
self._eviction_heap.clear() # MEDIUM FIX #16: Clear heap too
+ self._expiration_heap.clear()
self._stale_heap_entries = 0 # CRITICAL FIX: Reset stale counter
+ self._stale_expiration_heap_entries = 0
def _evict_if_needed(self) -> None:
- """Evict old entries if at capacity. Must be called with lock held.
+ """Evict expired, then oldest live entries until below capacity.
MEDIUM FIX #16: Use heap for O(log n) eviction instead of O(n) scan.
CRITICAL FIX: Track and clean stale heap entries to prevent memory leak.
"""
- # First, remove expired entries
- self._clean_expired()
-
- # CRITICAL FIX: Rebuild heap if too many stale entries
- # This prevents unbounded heap growth when entries are deleted/replaced
- heap_size = len(self._eviction_heap)
- if heap_size > 0:
- stale_ratio = self._stale_heap_entries / heap_size
- if stale_ratio >= self._heap_rebuild_threshold:
+ # Avoid the old full backend items() scan per new-key store().
+ # The expiration heap keeps cleanup proportional to expired roots.
+
+ backend_count = self._backend.count()
+ indexed_count = max(0, len(self._eviction_heap) - self._stale_heap_entries)
+ at_capacity = backend_count >= self._max_entries
+ if at_capacity:
+ backend_revision = self._read_backend_revision()
+ backend_changed = self._backend_revision_reader is not None and (
+ backend_revision is None or backend_revision != self._backend_revision
+ )
+ if indexed_count != backend_count or backend_changed:
self._rebuild_heap()
+ else:
+ self._rebuild_heap_if_needed()
+ if backend_revision is not None:
+ self._backend_revision = backend_revision
+ else:
+ self._rebuild_heap_if_needed()
+
+ # Remove expired entries first without reporting successful compression.
+ while self._expiration_heap:
+ expires_at, created_at, hash_key = self._expiration_heap[0]
+ if expires_at >= time.time():
+ break
+
+ entry = self._backend.get(hash_key)
+ if entry is None and at_capacity:
+ entry = self._backend.get(hash_key)
+ if entry is None:
+ return
+ heapq.heappop(self._expiration_heap)
+ if (
+ entry is not None
+ and entry.created_at == created_at
+ and self._expiration_time(entry) == expires_at
+ ):
+ self._backend.delete(hash_key)
+ self._stale_heap_entries += 1
+ elif self._stale_expiration_heap_entries > 0:
+ self._stale_expiration_heap_entries -= 1
+ else:
+ # Backend-side TTL purges leave the matching eviction tuple stale.
+ self._stale_heap_entries += 1
+
+ if not at_capacity or self._backend.count() < self._max_entries:
+ return
+
+ if not at_capacity:
+ return
# If still at capacity, remove oldest entries using heap
while self._backend.count() >= self._max_entries and self._eviction_heap:
@@ -756,19 +812,56 @@ class CompressionStore:
# (entry might have been deleted or replaced)
entry = self._backend.get(hash_key)
if entry is not None and entry.created_at == created_at:
- # HIGH FIX: Track eviction as "successful compression" if never retrieved
- # This prevents state divergence between store and feedback loop
- if self._enable_feedback and entry.retrieval_count == 0:
- # Entry was never retrieved = compression was successful
- # Notify feedback system so it knows this strategy worked
+ if not entry.is_expired() and self._enable_feedback and entry.retrieval_count == 0:
self._record_eviction_success(entry)
self._backend.delete(hash_key)
+ self._stale_expiration_heap_entries += 1
else:
# CRITICAL FIX: This was a stale entry, decrement counter
# (we already popped it, so the stale entry is now gone)
if self._stale_heap_entries > 0:
self._stale_heap_entries -= 1
+ def _mark_heap_entries_stale(self) -> None:
+ """Record lazy tuples invalidated by deleting or replacing an entry."""
+ self._stale_heap_entries += 1
+ self._stale_expiration_heap_entries += 1
+
+ @staticmethod
+ def _expiration_time(entry: CompressionEntry) -> float:
+ """Return the entry expiration time, saturating numeric overflow."""
+ try:
+ return entry.created_at + float(entry.ttl)
+ except OverflowError:
+ return float("inf") if entry.ttl > 0 else float("-inf")
+
+ def _read_backend_revision(self) -> object | None:
+ """Read optional backend revision metadata without breaking stores."""
+ if self._backend_revision_reader is None:
+ return None
+ try:
+ revision: object | None = self._backend_revision_reader()
+ return revision
+ except Exception:
+ logger.debug("CCR backend revision lookup failed", exc_info=True)
+ return None
+
+ def _rebuild_heap_if_needed(self) -> None:
+ """Rebuild both indexes when either stale ratio reaches the threshold."""
+ heap_size = len(self._eviction_heap)
+ expiration_heap_size = len(self._expiration_heap)
+ stale_ratio = self._stale_heap_entries / heap_size if heap_size else 0
+ stale_expiration_ratio = (
+ self._stale_expiration_heap_entries / expiration_heap_size
+ if expiration_heap_size
+ else 0
+ )
+ if (
+ stale_ratio >= self._heap_rebuild_threshold
+ or stale_expiration_ratio >= self._heap_rebuild_threshold
+ ):
+ self._rebuild_heap()
+
def _clean_expired(self) -> None:
"""Remove expired entries. Must be called with lock held.
@@ -782,9 +875,7 @@ class CompressionStore:
expired_keys = [key for key, entry in self._backend.items() if entry.is_expired()]
for key in expired_keys:
self._backend.delete(key)
- # CRITICAL FIX: Increment stale counter - the heap still has an entry
- # for this key that will be stale when we try to evict
- self._stale_heap_entries += 1
+ self._mark_heap_entries_stale()
def _rebuild_heap(self) -> None:
"""Rebuild heap from current store entries. Must be called with lock held.
@@ -792,15 +883,20 @@ class CompressionStore:
CRITICAL FIX: This removes stale heap entries that accumulate when entries
are deleted or replaced. Without this, the heap grows unboundedly.
"""
- # Build new heap from current store entries only
- self._eviction_heap = [
- (entry.created_at, hash_key) for hash_key, entry in self._backend.items()
+ # Reuse one snapshot so rebuilding never doubles backend enumeration.
+ entries = self._backend.items()
+ self._eviction_heap = [(entry.created_at, hash_key) for hash_key, entry in entries]
+ self._expiration_heap = [
+ (self._expiration_time(entry), entry.created_at, hash_key)
+ for hash_key, entry in entries
]
heapq.heapify(self._eviction_heap)
+ heapq.heapify(self._expiration_heap)
# Reset stale counter - heap is now clean
self._stale_heap_entries = 0
+ self._stale_expiration_heap_entries = 0
logger.debug(
- "Rebuilt eviction heap: %d entries",
+ "Rebuilt eviction heaps: %d entries",
len(self._eviction_heap),
)
diff --git a/tests/test_compression_store.py b/tests/test_compression_store.py
index 6d9bfdc2b..9f709d3b6 100644
--- a/tests/test_compression_store.py
+++ b/tests/test_compression_store.py
@@ -26,6 +26,8 @@ from unittest.mock import MagicMock, patch
import pytest
+from headroom.cache.backends.memory import InMemoryBackend
+from headroom.cache.backends.sqlite import SQLiteBackend
from headroom.cache.compression_store import (
CCR_TTL_SECONDS_ENV,
DEFAULT_CCR_TTL_SECONDS,
@@ -662,6 +664,126 @@ class TestCompressionStoreTTL:
class TestCompressionStoreEviction:
"""Tests for CompressionStore memory limits and eviction."""
+ def test_new_key_store_does_not_scan_backend_for_expiry(self):
+ """New-key store() must not trigger a backend items() expiry scan."""
+
+ class _ItemsCountingBackend(InMemoryBackend):
+ def __init__(self) -> None:
+ super().__init__()
+ self.items_calls = 0
+
+ def items(self) -> list[tuple[str, CompressionEntry]]:
+ self.items_calls += 1
+ return super().items()
+
+ backend = _ItemsCountingBackend()
+ store = CompressionStore(max_entries=2, backend=backend)
+
+ for i in range(3):
+ store.store(original=f"content_{i}", compressed=f"compressed_{i}")
+
+ assert backend.items_calls == 0
+
+ def test_expired_cleanup_below_capacity_is_bounded(self):
+ """A new key cleans at most one expired entry while capacity is available."""
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=10, enable_feedback=False, backend=backend)
+
+ with (
+ patch("headroom.cache.compression_store.time.time") as now,
+ patch.object(backend, "get", wraps=backend.get) as get,
+ patch.object(backend, "delete", wraps=backend.delete) as delete,
+ patch.object(backend, "items", wraps=backend.items) as items,
+ patch.object(backend, "count", wraps=backend.count) as count,
+ ):
+ operations = (get, delete, items, count)
+ now.return_value = 100
+ for i in range(4):
+ store.store(original=f"expired_{i}", compressed="expired", ttl=1)
+
+ for operation in operations:
+ operation.reset_mock()
+ now.return_value = 200
+ store.store(original="new_1", compressed="new", ttl=100)
+ assert tuple(operation.call_count for operation in operations) == (2, 1, 0, 1)
+
+ for operation in operations:
+ operation.reset_mock()
+ now.return_value = 201
+ store.store(original="new_2", compressed="new", ttl=100)
+ assert (items.call_count, count.call_count) == (0, 1)
+
+ def test_in_memory_oversized_ttl_at_capacity_keeps_indexes_consistent(self):
+ """An oversized positive TTL is stored as a non-expiring heap entry."""
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=1, enable_feedback=False, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 100.0
+ old_hash = store.store(original="old", compressed="old")
+ now.return_value = 101.0
+ oversized_hash = store.store(
+ original="oversized",
+ compressed="oversized",
+ ttl=10**1000,
+ )
+
+ entry = backend.get(oversized_hash)
+ assert entry is not None
+ assert not backend.exists(old_hash)
+ assert len(store._eviction_heap) - store._stale_heap_entries == backend.count() == 1
+ assert len(store._expiration_heap) - store._stale_expiration_heap_entries == backend.count()
+ assert (entry.created_at, oversized_hash) in store._eviction_heap
+ assert (float("inf"), entry.created_at, oversized_hash) in store._expiration_heap
+
+ @pytest.mark.parametrize("mode", ["count", "revision", "revision_error", "stale_revision"])
+ def test_shared_sqlite_pressure_reconciles_sibling_entries(self, tmp_path, mode: str):
+ """Capacity reconciliation covers sibling count and identity changes."""
+
+ class _RevisionFailureSQLiteBackend(SQLiteBackend):
+ fail_revision = False
+
+ def external_revision(self) -> int | None:
+ if self.fail_revision:
+ raise RuntimeError("revision unavailable")
+ return super().external_revision()
+
+ db_path = tmp_path / "ccr.db"
+ backend = _RevisionFailureSQLiteBackend(db_path)
+ sibling_backend = SQLiteBackend(db_path)
+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)
+ sibling_store = CompressionStore(enable_feedback=False, backend=sibling_backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 90
+ removed_hash = None
+ if mode != "count":
+ removed_hash = store.store(
+ original="removed",
+ compressed="removed",
+ ttl=1 if mode == "stale_revision" else 100,
+ )
+ now.return_value = 100
+ live_hash = store.store(
+ original="live",
+ compressed="live",
+ ttl=100,
+ tool_signature_hash="live_signature",
+ compression_strategy="top_k",
+ )
+ if removed_hash is not None and mode != "stale_revision":
+ sibling_backend.delete(removed_hash)
+ now.return_value = 110
+ sibling_store.store(original="expired", compressed="expired", ttl=5)
+ backend.fail_revision = mode == "revision_error"
+ now.return_value = 116
+ with patch.object(backend, "items", wraps=backend.items) as items:
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ assert set(backend.keys()) == {live_hash, new_hash}
+ assert items.call_count == 1
+ assert store._pending_feedback_events == []
+
def test_eviction_at_capacity(self, store_with_small_capacity: CompressionStore):
"""Oldest entries are evicted when at capacity."""
hashes = []
@@ -708,6 +830,24 @@ class TestCompressionStoreEviction:
assert store_with_small_capacity.exists(hashes[2])
assert store_with_small_capacity.exists(new_hash)
+ def test_mixed_ttl_pressure_evicts_oldest_created_live_entry(self):
+ """Live eviction remains creation-ordered across mixed TTLs."""
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 100
+ older_long_ttl = store.store(original="older", compressed="older", ttl=100)
+ now.return_value = 110
+ newer_short_ttl = store.store(original="newer", compressed="newer", ttl=20)
+ now.return_value = 120
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ assert not backend.exists(older_long_ttl)
+ assert backend.exists(newer_short_ttl)
+ assert backend.exists(new_hash)
+ assert backend.count() == 2
+
def test_duplicate_store_at_capacity_does_not_evict(
self, store_with_small_capacity: CompressionStore
):
@@ -736,25 +876,109 @@ class TestCompressionStoreEviction:
def test_eviction_cleans_expired_first(self):
"""Eviction cleans expired entries before evicting valid ones."""
- store = CompressionStore(max_entries=3, default_ttl=1)
-
- # Add 2 entries that will expire
- hash1 = store.store(original="content_1", compressed="c1", ttl=1)
- hash2 = store.store(original="content_2", compressed="c2", ttl=1)
-
- time.sleep(1.1) # Wait for expiration
-
- # Add 2 more entries (should clean expired first, not evict new)
- hash3 = store.store(original="content_3", compressed="c3", ttl=300)
- hash4 = store.store(original="content_4", compressed="c4", ttl=300)
-
- # Expired entries should be gone
- assert not store.exists(hash1)
- assert not store.exists(hash2)
-
- # New entries should exist
- assert store.exists(hash3)
- assert store.exists(hash4)
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 100
+ live_hash = store.store(original="live", compressed="live", ttl=100)
+ now.return_value = 110
+ expired_hash = store.store(
+ original="expired",
+ compressed="expired",
+ ttl=5,
+ tool_signature_hash="expired_signature",
+ compression_strategy="top_k",
+ )
+ now.return_value = 116
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ assert backend.exists(live_hash)
+ assert not backend.exists(expired_hash)
+ assert backend.exists(new_hash)
+ assert backend.count() == 2
+ assert store._pending_feedback_events == []
+
+ @pytest.mark.parametrize("misses", [1, 2])
+ def test_expiration_cleanup_handles_transient_backend_misses(self, misses: int):
+ """Expiry lookup misses retry once, then fail open without live eviction."""
+
+ class _TransientMissBackend(InMemoryBackend):
+ miss_key: str | None = None
+ misses_remaining = 0
+
+ def get(self, hash_key: str) -> CompressionEntry | None:
+ if hash_key == self.miss_key and self.misses_remaining:
+ self.misses_remaining -= 1
+ return None
+ return super().get(hash_key)
+
+ backend = _TransientMissBackend()
+ store = CompressionStore(max_entries=2, enable_feedback=True, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ if misses == 1:
+ now.return_value = 80
+ store.store(original="stale", compressed="stale", ttl=100)
+ now.return_value = 90
+ live_hash = store.store(
+ original="live",
+ compressed="live",
+ ttl=100,
+ tool_signature_hash="live_signature",
+ compression_strategy="top_k",
+ )
+ now.return_value = 91
+ expired_hash = store.store(original="expired", compressed="expired", ttl=5)
+ backend.miss_key = expired_hash
+ backend.misses_remaining = misses
+ now.return_value = 97
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ expected_keys = {live_hash, new_hash}
+ if misses == 2:
+ expected_keys.add(expired_hash)
+ assert set(backend.keys()) == expected_keys
+ assert store._pending_feedback_events == []
+
+ def test_eviction_ignores_stale_expiration_after_replacement(self):
+ """A replaced entry survives its old expiration heap tuple."""
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 90
+ oldest_hash = store.store(original="oldest", compressed="oldest", ttl=100)
+ now.return_value = 100
+ replacement_hash = store.store(original="replacement", compressed="v1", ttl=5)
+ now.return_value = 101
+ store.store(original="replacement", compressed="v2", ttl=100)
+ now.return_value = 110
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ assert not backend.exists(oldest_hash)
+ assert backend.exists(replacement_hash)
+ assert backend.exists(new_hash)
+
+ def test_eviction_ignores_stale_indexes_after_read_cleanup(self):
+ """Read-side expiry cleanup leaves both eviction indexes consistent."""
+ backend = InMemoryBackend()
+ store = CompressionStore(max_entries=2, enable_feedback=False, backend=backend)
+
+ with patch("headroom.cache.compression_store.time.time") as now:
+ now.return_value = 100
+ expired_hash = store.store(original="expired", compressed="expired", ttl=1)
+ now.return_value = 102
+ assert store.retrieve(expired_hash) is None
+ first_live = store.store(original="first", compressed="first", ttl=100)
+ now.return_value = 103
+ second_live = store.store(original="second", compressed="second", ttl=100)
+ now.return_value = 104
+ new_hash = store.store(original="new", compressed="new", ttl=100)
+
+ assert not backend.exists(first_live)
+ assert backend.exists(second_live)
+ assert backend.exists(new_hash)
def test_heap_rebuild_on_stale_threshold(self):
"""Heap is rebuilt when stale entry ratio exceeds threshold."""
@@ -768,8 +992,10 @@ class TestCompressionStoreEviction:
for i in range(5):
store.store(original=f"content_{i}", compressed=f"updated_{i}")
- # Stale ratio should be tracked
- # The heap rebuild happens automatically when threshold is exceeded
+ assert len(store._eviction_heap) == 5
+ assert len(store._expiration_heap) == 5
+ assert store._stale_heap_entries == 0
+ assert store._stale_expiration_heap_entries == 0
# =============================================================================
@@ -1054,6 +1280,10 @@ class TestCompressionStoreEdgeCases:
stats = store.get_stats()
assert stats["entry_count"] == 0
+ assert store._eviction_heap == []
+ assert store._expiration_heap == []
+ assert store._stale_heap_entries == 0
+ assert store._stale_expiration_heap_entries == 0
def test_clear_removes_retrieval_events(self, store: CompressionStore):
"""clear() removes retrieval events."""
@@ -1263,7 +1493,7 @@ class TestCompressionStoreFeedback:
store = CompressionStore(max_entries=2, enable_feedback=True)
# Store entries with signature hash for eviction tracking
- store.store(
+ first_hash = store.store(
original="content_0",
compressed="c0",
tool_signature_hash="sig_0",
@@ -1290,6 +1520,9 @@ class TestCompressionStoreFeedback:
# The evicted entry (content_0) was never retrieved,
# so an eviction_success event should be queued
# (tested via the pending_feedback mechanism)
+ assert len(store._pending_feedback_events) == 1
+ assert store._pending_feedback_events[0].hash == first_hash
+ assert store._pending_feedback_events[0].retrieval_type == "eviction_success"
# =============================================================================