Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(batch): preserve live contexts during capacity cleanup (#4210)

Mansurshakh Japarov2cbdd22a6578
diff --git a/headroom/ccr/batch_store.py b/headroom/ccr/batch_store.py
index a68779a99..88377116a 100644
--- a/headroom/ccr/batch_store.py
+++ b/headroom/ccr/batch_store.py
@@ -91,9 +91,9 @@ class BatchContextStore:
Features:
- TTL-based expiration
- - Automatic cleanup of expired entries
+ - Expired entries reclaimed before live contexts are evicted at capacity
- Thread-safe operations
- - Memory limits
+ - Memory limits; replacing a batch refreshes its TTL without evicting another batch
Usage:
store = BatchContextStore()
@@ -135,7 +135,7 @@ class BatchContextStore:
"""
async with self._lock:
# Enforce memory limit
- if len(self._contexts) >= self._max_contexts:
+ if context.batch_id not in self._contexts and len(self._contexts) >= self._max_contexts:
# Remove oldest entries
await self._cleanup_oldest()
@@ -205,10 +205,15 @@ class BatchContextStore:
async def _cleanup_oldest(self) -> None:
"""Remove oldest entries to make room for new ones."""
- # Sort by creation time, remove oldest 10%
- if not self._contexts:
+ now = time.time()
+ expired = [batch_id for batch_id, ctx in self._contexts.items() if ctx.expires_at < now]
+ for batch_id in expired:
+ del self._contexts[batch_id]
+ if len(self._contexts) < self._max_contexts:
return
+ # Sort remaining live contexts by creation time, remove oldest 10%.
+
sorted_entries = sorted(
self._contexts.items(),
key=lambda x: x[1].created_at,
diff --git a/tests/test_ccr_batch_store.py b/tests/test_ccr_batch_store.py
index a4ab6bbec..0061ef747 100644
--- a/tests/test_ccr_batch_store.py
+++ b/tests/test_ccr_batch_store.py
@@ -58,6 +58,52 @@ async def test_batch_context_store_core_operations(monkeypatch) -> None:
assert await store.remove("b2") is False
+async def test_replacing_full_store_context_preserves_other_batch_and_refreshes_ttl(
+ monkeypatch,
+) -> None:
+ now = {"value": 100.0}
+ monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])
+ store = BatchContextStore(ttl=10, max_contexts=2)
+ first = BatchContext(batch_id="first", provider="openai", created_at=0)
+ second = BatchContext(batch_id="second", provider="openai", created_at=1)
+ await store.store(first)
+ await store.store(second)
+
+ now["value"] = 105.0
+ replacement = BatchContext(batch_id="second", provider="openai")
+ replacement.add_request(
+ BatchRequestContext(custom_id="request", messages=[], model="updated-model")
+ )
+ await store.store(replacement)
+
+ assert await store.get("first") is first
+ assert (await store.get("second")).get_request("request").model == "updated-model"
+ now["value"] = 111.0
+ assert await store.get("first") is None
+ assert await store.get("second") is replacement
+
+
+async def test_full_store_reclaims_expired_batch_before_evicting_live_batch(
+ monkeypatch,
+) -> None:
+ now = {"value": 100.0}
+ monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])
+ store = BatchContextStore(ttl=10, max_contexts=2)
+ expired = BatchContext(batch_id="expired", provider="openai", created_at=1)
+ await store.store(expired)
+ now["value"] = 105.0
+ live = BatchContext(batch_id="live", provider="openai", created_at=0)
+ await store.store(live)
+
+ now["value"] = 111.0
+ new = BatchContext(batch_id="new", provider="openai", created_at=2)
+ await store.store(new)
+
+ assert await store.get("live") is live
+ assert await store.get("expired") is None
+ assert await store.get("new") is new
+
+
async def test_batch_context_store_cleanup_stats_and_memory_stats(monkeypatch) -> None:
now = {"value": 200.0}
monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])