Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(batch): preserve live contexts during capacity cleanup (#4210)
Mansurshakh Japarov2cbdd22a6578
diff --git a/headroom/ccr/batch_store.py b/headroom/ccr/batch_store.pyindex a68779a99..88377116a 100644--- a/headroom/ccr/batch_store.py+++ b/headroom/ccr/batch_store.py@@ -91,9 +91,9 @@ class BatchContextStore:Features:- TTL-based expiration- - Automatic cleanup of expired entries+ - Expired entries reclaimed before live contexts are evicted at capacity- Thread-safe operations- - Memory limits+ - Memory limits; replacing a batch refreshes its TTL without evicting another batchUsage:store = BatchContextStore()@@ -135,7 +135,7 @@ class BatchContextStore:"""async with self._lock:# Enforce memory limit- if len(self._contexts) >= self._max_contexts:+ if context.batch_id not in self._contexts and len(self._contexts) >= self._max_contexts:# Remove oldest entriesawait self._cleanup_oldest()@@ -205,10 +205,15 @@ class BatchContextStore:async def _cleanup_oldest(self) -> None:"""Remove oldest entries to make room for new ones."""- # Sort by creation time, remove oldest 10%- if not self._contexts:+ now = time.time()+ expired = [batch_id for batch_id, ctx in self._contexts.items() if ctx.expires_at < now]+ for batch_id in expired:+ del self._contexts[batch_id]+ if len(self._contexts) < self._max_contexts:return+ # Sort remaining live contexts by creation time, remove oldest 10%.+sorted_entries = sorted(self._contexts.items(),key=lambda x: x[1].created_at,diff --git a/tests/test_ccr_batch_store.py b/tests/test_ccr_batch_store.pyindex a4ab6bbec..0061ef747 100644--- a/tests/test_ccr_batch_store.py+++ b/tests/test_ccr_batch_store.py@@ -58,6 +58,52 @@ async def test_batch_context_store_core_operations(monkeypatch) -> None:assert await store.remove("b2") is False+async def test_replacing_full_store_context_preserves_other_batch_and_refreshes_ttl(+ monkeypatch,+) -> None:+ now = {"value": 100.0}+ monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])+ store = BatchContextStore(ttl=10, max_contexts=2)+ first = BatchContext(batch_id="first", provider="openai", created_at=0)+ second = BatchContext(batch_id="second", provider="openai", created_at=1)+ await store.store(first)+ await store.store(second)++ now["value"] = 105.0+ replacement = BatchContext(batch_id="second", provider="openai")+ replacement.add_request(+ BatchRequestContext(custom_id="request", messages=[], model="updated-model")+ )+ await store.store(replacement)++ assert await store.get("first") is first+ assert (await store.get("second")).get_request("request").model == "updated-model"+ now["value"] = 111.0+ assert await store.get("first") is None+ assert await store.get("second") is replacement+++async def test_full_store_reclaims_expired_batch_before_evicting_live_batch(+ monkeypatch,+) -> None:+ now = {"value": 100.0}+ monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])+ store = BatchContextStore(ttl=10, max_contexts=2)+ expired = BatchContext(batch_id="expired", provider="openai", created_at=1)+ await store.store(expired)+ now["value"] = 105.0+ live = BatchContext(batch_id="live", provider="openai", created_at=0)+ await store.store(live)++ now["value"] = 111.0+ new = BatchContext(batch_id="new", provider="openai", created_at=2)+ await store.store(new)++ assert await store.get("live") is live+ assert await store.get("expired") is None+ assert await store.get("new") is new++async def test_batch_context_store_cleanup_stats_and_memory_stats(monkeypatch) -> None:now = {"value": 200.0}monkeypatch.setattr("headroom.ccr.batch_store.time.time", lambda: now["value"])