Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): keep the in-memory order queue from evicting re-stored entries (#4545)
HetCreepc6f5bf9549e9
diff --git a/crates/headroom-core/src/ccr/backends/in_memory.rs b/crates/headroom-core/src/ccr/backends/in_memory.rsindex d14b04d3d..2b7563ffd 100644--- a/crates/headroom-core/src/ccr/backends/in_memory.rs+++ b/crates/headroom-core/src/ccr/backends/in_memory.rs@@ -10,6 +10,7 @@//! workers (see `RUST_DEV.md` "Multi-worker deployment").use std::collections::VecDeque;+use std::sync::atomic::{AtomicU64, Ordering};use std::sync::Mutex;use std::time::{Duration, Instant};@@ -33,11 +34,18 @@ use crate::ccr::{max_lifetime_for, CcrStore, DEFAULT_CAPACITY, DEFAULT_TTL};/// small sweep.pub struct InMemoryCcrStore {map: DashMap<String, Entry>,- /// FIFO insertion order. Stale entries (already removed from `map`- /// via TTL expiry) are tolerated — `pop_front` + `map.remove` is a- /// no-op for missing keys, and capacity-bounded sweeps loop until- /// they actually evict a real entry.- order: Mutex<VecDeque<String>>,+ /// FIFO insertion order as `(hash, seq)`. A queue item is live only+ /// while `map[hash].seq == seq`: an entry removed by TTL expiry and+ /// stored again later gets a new `seq`, so its old queue item can+ /// neither evict the fresh entry nor pin it. Stale items are skipped+ /// by eviction and dropped by `compact_order` once the queue grows+ /// past twice the capacity.+ order: Mutex<VecDeque<(String, u64)>>,+ next_seq: AtomicU64,+ /// Test-only pause point between inserting a fresh entry and+ /// recording it in `order`, to reproduce writer interleavings.+ #[cfg(test)]+ after_insert_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,ttl: Duration,max_lifetime: Duration,capacity: usize,@@ -46,6 +54,7 @@ pub struct InMemoryCcrStore {#[derive(Clone)]struct Entry {payload: String,+ seq: u64,inserted: Instant,last_accessed: Instant,}@@ -77,25 +86,37 @@ impl InMemoryCcrStore {Self {map: DashMap::with_capacity(capacity),order: Mutex::new(VecDeque::with_capacity(capacity)),+ next_seq: AtomicU64::new(0),+ #[cfg(test)]+ after_insert_hook: Mutex::new(None),ttl,max_lifetime,capacity,}}- /// Sweep the order queue, dropping leading entries that no longer- /// exist in the map (already expired or evicted), then evict- /// real entries until `map.len() < capacity`. Called only from- /// `put` on a fresh-key insert path.- fn evict_until_under_capacity(&self) {- let mut guard = self.order.lock().expect("ccr order mutex poisoned");+ /// Evict real entries, oldest first, until `map.len() < capacity`.+ /// The caller holds the `order` lock, which serializes every+ /// fresh-key insert, so no entry can be in the map without its+ /// queue item while this runs.+ fn evict_until_under_capacity(&self, guard: &mut VecDeque<(String, u64)>) {while self.map.len() >= self.capacity {- let Some(oldest) = guard.pop_front() else {+ let Some((oldest, seq)) = guard.pop_front() else {break;};- // `remove` is a no-op if `oldest` was already lazy-expired.- // Loop continues until we actually shrink the map.- self.map.remove(&oldest);+ // Remove only the entry this queue item was recorded for. A+ // stale item (the entry expired, or was stored again since)+ // removes nothing, and the loop moves on to the next one.+ self.map.remove_if(&oldest, |_, entry| entry.seq == seq);+ }+ }++ /// Drop queue items whose entry is gone or was re-stored, so the+ /// queue stays within a constant factor of the live entries even+ /// when entries keep expiring through `get` without any eviction.+ fn compact_order(&self, order: &mut VecDeque<(String, u64)>) {+ if order.len() > self.capacity.max(16) * 2 {+ order.retain(|(hash, seq)| self.map.get(hash).is_some_and(|e| e.seq == *seq));}}}@@ -119,28 +140,39 @@ impl CcrStore for InMemoryCcrStore {return;}- // New entry. Cap-bound first (may sweep a few stale order- // entries), then insert and append to the FIFO queue.+ // New entry. The capacity check, the insert and the queue append+ // happen under one `order` lock, so a concurrent fresh insert can+ // never sit in the map without its queue item (which would let+ // eviction run out of queue while the map is still full). Lock+ // order is always `order` then a map shard; no path takes `order`+ // while holding a shard guard.+ let mut order = self.order.lock().expect("ccr order mutex poisoned");+ if let Some(mut existing) = self.map.get_mut(hash) {+ // Another writer stored this hash while we waited for the lock.+ let now = Instant::now();+ existing.payload = payload.to_string();+ existing.inserted = now;+ existing.last_accessed = now;+ return;+ }if self.map.len() >= self.capacity {- self.evict_until_under_capacity();+ self.evict_until_under_capacity(&mut order);}let now = Instant::now();+ let seq = self.next_seq.fetch_add(1, Ordering::Relaxed);let entry = Entry {payload: payload.to_string(),+ seq,inserted: now,last_accessed: now,};- let prev = self.map.insert(hash.to_string(), entry);- if prev.is_none() {- // Truly new key — record in FIFO order. (If `prev.is_some()`- // it means another thread re-inserted between our get_mut- // miss and this insert; treat that as a fast-path overwrite- // and skip the queue append to avoid duplicates.)- self.order- .lock()- .expect("ccr order mutex poisoned")- .push_back(hash.to_string());+ self.map.insert(hash.to_string(), entry);+ #[cfg(test)]+ if let Some(hook) = self.after_insert_hook.lock().expect("hook mutex").take() {+ hook();}+ order.push_back((hash.to_string(), seq));+ self.compact_order(&mut order);}fn get(&self, hash: &str) -> Option<String> {@@ -238,6 +270,80 @@ mod tests {assert_eq!(store.len(), 0);}+ #[test]+ fn reput_after_expiry_keeps_the_newest_entry() {+ // a expires and is dropped by get(); b is stored; a is stored again+ // (now the newest entry). At capacity, the oldest live entry is b,+ // so storing c must evict b and keep the fresh a.+ let store = InMemoryCcrStore::with_capacity_and_ttl(2, Duration::from_millis(30));+ store.put("a", "old");+ std::thread::sleep(Duration::from_millis(60));+ assert_eq!(store.get("a"), None);+ store.put("b", "2");+ store.put("a", "fresh");+ store.put("c", "3");+ let a = store.get("a");+ assert_eq!(store.get("b"), None, "b was the oldest live entry");+ assert_eq!(+ a.as_deref(),+ Some("fresh"),+ "fresh a was evicted instead of b"+ );+ }++ #[test]+ fn order_queue_stays_bounded_when_entries_expire_through_get() {+ let store = InMemoryCcrStore::with_capacity_and_ttl(4, Duration::from_millis(1));+ for i in 0..2000 {+ let k = format!("k{i}");+ store.put(&k, "v");+ std::thread::sleep(Duration::from_millis(2));+ assert_eq!(store.get(&k), None);+ }+ let n = store.order.lock().unwrap().len();+ assert!(+ n <= 32,+ "order queue grew to {n} while the map held {}",+ store.len()+ );+ }++ #[test]+ fn concurrent_fresh_inserts_never_exceed_capacity() {+ use std::sync::{mpsc, Arc};+ // Capacity 1. Writer A stores x and pauses after the map insert,+ // before x is recorded in the order queue. Writer C then stores y.+ // If C could run in that gap it would find no queue item to evict+ // and leave both x and y in a capacity-1 store.+ let store = Arc::new(InMemoryCcrStore::with_capacity_and_ttl(1, DEFAULT_TTL));+ let (paused_tx, paused_rx) = mpsc::channel::<()>();+ let (release_tx, release_rx) = mpsc::channel::<()>();+ *store.after_insert_hook.lock().unwrap() = Some(Box::new(move || {+ paused_tx.send(()).unwrap();+ release_rx.recv().unwrap();+ }));+ let a = {+ let s = Arc::clone(&store);+ std::thread::spawn(move || s.put("x", "1"))+ };+ paused_rx.recv().unwrap();+ let c = {+ let s = Arc::clone(&store);+ std::thread::spawn(move || s.put("y", "2"))+ };+ // Give C time to run into the gap if it can.+ std::thread::sleep(Duration::from_millis(200));+ release_tx.send(()).unwrap();+ a.join().unwrap();+ c.join().unwrap();+ assert!(+ store.len() <= 1,+ "capacity 1 store holds {} entries",+ store.len()+ );+ assert_eq!(store.get("y").as_deref(), Some("2"));+ }+#[test]fn store_is_send_sync() {fn assert_send_sync<T: Send + Sync>() {}