Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(ccr): keep the in-memory order queue from evicting re-stored entries (#4545)

HetCreepc6f5bf9549e9
diff --git a/crates/headroom-core/src/ccr/backends/in_memory.rs b/crates/headroom-core/src/ccr/backends/in_memory.rs
index d14b04d3d..2b7563ffd 100644
--- a/crates/headroom-core/src/ccr/backends/in_memory.rs
+++ b/crates/headroom-core/src/ccr/backends/in_memory.rs
@@ -10,6 +10,7 @@
//! workers (see `RUST_DEV.md` "Multi-worker deployment").
use std::collections::VecDeque;
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
use std::time::{Duration, Instant};
@@ -33,11 +34,18 @@ use crate::ccr::{max_lifetime_for, CcrStore, DEFAULT_CAPACITY, DEFAULT_TTL};
/// small sweep.
pub struct InMemoryCcrStore {
map: DashMap<String, Entry>,
- /// FIFO insertion order. Stale entries (already removed from `map`
- /// via TTL expiry) are tolerated — `pop_front` + `map.remove` is a
- /// no-op for missing keys, and capacity-bounded sweeps loop until
- /// they actually evict a real entry.
- order: Mutex<VecDeque<String>>,
+ /// FIFO insertion order as `(hash, seq)`. A queue item is live only
+ /// while `map[hash].seq == seq`: an entry removed by TTL expiry and
+ /// stored again later gets a new `seq`, so its old queue item can
+ /// neither evict the fresh entry nor pin it. Stale items are skipped
+ /// by eviction and dropped by `compact_order` once the queue grows
+ /// past twice the capacity.
+ order: Mutex<VecDeque<(String, u64)>>,
+ next_seq: AtomicU64,
+ /// Test-only pause point between inserting a fresh entry and
+ /// recording it in `order`, to reproduce writer interleavings.
+ #[cfg(test)]
+ after_insert_hook: Mutex<Option<Box<dyn FnOnce() + Send>>>,
ttl: Duration,
max_lifetime: Duration,
capacity: usize,
@@ -46,6 +54,7 @@ pub struct InMemoryCcrStore {
#[derive(Clone)]
struct Entry {
payload: String,
+ seq: u64,
inserted: Instant,
last_accessed: Instant,
}
@@ -77,25 +86,37 @@ impl InMemoryCcrStore {
Self {
map: DashMap::with_capacity(capacity),
order: Mutex::new(VecDeque::with_capacity(capacity)),
+ next_seq: AtomicU64::new(0),
+ #[cfg(test)]
+ after_insert_hook: Mutex::new(None),
ttl,
max_lifetime,
capacity,
}
}
- /// Sweep the order queue, dropping leading entries that no longer
- /// exist in the map (already expired or evicted), then evict
- /// real entries until `map.len() < capacity`. Called only from
- /// `put` on a fresh-key insert path.
- fn evict_until_under_capacity(&self) {
- let mut guard = self.order.lock().expect("ccr order mutex poisoned");
+ /// Evict real entries, oldest first, until `map.len() < capacity`.
+ /// The caller holds the `order` lock, which serializes every
+ /// fresh-key insert, so no entry can be in the map without its
+ /// queue item while this runs.
+ fn evict_until_under_capacity(&self, guard: &mut VecDeque<(String, u64)>) {
while self.map.len() >= self.capacity {
- let Some(oldest) = guard.pop_front() else {
+ let Some((oldest, seq)) = guard.pop_front() else {
break;
};
- // `remove` is a no-op if `oldest` was already lazy-expired.
- // Loop continues until we actually shrink the map.
- self.map.remove(&oldest);
+ // Remove only the entry this queue item was recorded for. A
+ // stale item (the entry expired, or was stored again since)
+ // removes nothing, and the loop moves on to the next one.
+ self.map.remove_if(&oldest, |_, entry| entry.seq == seq);
+ }
+ }
+
+ /// Drop queue items whose entry is gone or was re-stored, so the
+ /// queue stays within a constant factor of the live entries even
+ /// when entries keep expiring through `get` without any eviction.
+ fn compact_order(&self, order: &mut VecDeque<(String, u64)>) {
+ if order.len() > self.capacity.max(16) * 2 {
+ order.retain(|(hash, seq)| self.map.get(hash).is_some_and(|e| e.seq == *seq));
}
}
}
@@ -119,28 +140,39 @@ impl CcrStore for InMemoryCcrStore {
return;
}
- // New entry. Cap-bound first (may sweep a few stale order
- // entries), then insert and append to the FIFO queue.
+ // New entry. The capacity check, the insert and the queue append
+ // happen under one `order` lock, so a concurrent fresh insert can
+ // never sit in the map without its queue item (which would let
+ // eviction run out of queue while the map is still full). Lock
+ // order is always `order` then a map shard; no path takes `order`
+ // while holding a shard guard.
+ let mut order = self.order.lock().expect("ccr order mutex poisoned");
+ if let Some(mut existing) = self.map.get_mut(hash) {
+ // Another writer stored this hash while we waited for the lock.
+ let now = Instant::now();
+ existing.payload = payload.to_string();
+ existing.inserted = now;
+ existing.last_accessed = now;
+ return;
+ }
if self.map.len() >= self.capacity {
- self.evict_until_under_capacity();
+ self.evict_until_under_capacity(&mut order);
}
let now = Instant::now();
+ let seq = self.next_seq.fetch_add(1, Ordering::Relaxed);
let entry = Entry {
payload: payload.to_string(),
+ seq,
inserted: now,
last_accessed: now,
};
- let prev = self.map.insert(hash.to_string(), entry);
- if prev.is_none() {
- // Truly new key — record in FIFO order. (If `prev.is_some()`
- // it means another thread re-inserted between our get_mut
- // miss and this insert; treat that as a fast-path overwrite
- // and skip the queue append to avoid duplicates.)
- self.order
- .lock()
- .expect("ccr order mutex poisoned")
- .push_back(hash.to_string());
+ self.map.insert(hash.to_string(), entry);
+ #[cfg(test)]
+ if let Some(hook) = self.after_insert_hook.lock().expect("hook mutex").take() {
+ hook();
}
+ order.push_back((hash.to_string(), seq));
+ self.compact_order(&mut order);
}
fn get(&self, hash: &str) -> Option<String> {
@@ -238,6 +270,80 @@ mod tests {
assert_eq!(store.len(), 0);
}
+ #[test]
+ fn reput_after_expiry_keeps_the_newest_entry() {
+ // a expires and is dropped by get(); b is stored; a is stored again
+ // (now the newest entry). At capacity, the oldest live entry is b,
+ // so storing c must evict b and keep the fresh a.
+ let store = InMemoryCcrStore::with_capacity_and_ttl(2, Duration::from_millis(30));
+ store.put("a", "old");
+ std::thread::sleep(Duration::from_millis(60));
+ assert_eq!(store.get("a"), None);
+ store.put("b", "2");
+ store.put("a", "fresh");
+ store.put("c", "3");
+ let a = store.get("a");
+ assert_eq!(store.get("b"), None, "b was the oldest live entry");
+ assert_eq!(
+ a.as_deref(),
+ Some("fresh"),
+ "fresh a was evicted instead of b"
+ );
+ }
+
+ #[test]
+ fn order_queue_stays_bounded_when_entries_expire_through_get() {
+ let store = InMemoryCcrStore::with_capacity_and_ttl(4, Duration::from_millis(1));
+ for i in 0..2000 {
+ let k = format!("k{i}");
+ store.put(&k, "v");
+ std::thread::sleep(Duration::from_millis(2));
+ assert_eq!(store.get(&k), None);
+ }
+ let n = store.order.lock().unwrap().len();
+ assert!(
+ n <= 32,
+ "order queue grew to {n} while the map held {}",
+ store.len()
+ );
+ }
+
+ #[test]
+ fn concurrent_fresh_inserts_never_exceed_capacity() {
+ use std::sync::{mpsc, Arc};
+ // Capacity 1. Writer A stores x and pauses after the map insert,
+ // before x is recorded in the order queue. Writer C then stores y.
+ // If C could run in that gap it would find no queue item to evict
+ // and leave both x and y in a capacity-1 store.
+ let store = Arc::new(InMemoryCcrStore::with_capacity_and_ttl(1, DEFAULT_TTL));
+ let (paused_tx, paused_rx) = mpsc::channel::<()>();
+ let (release_tx, release_rx) = mpsc::channel::<()>();
+ *store.after_insert_hook.lock().unwrap() = Some(Box::new(move || {
+ paused_tx.send(()).unwrap();
+ release_rx.recv().unwrap();
+ }));
+ let a = {
+ let s = Arc::clone(&store);
+ std::thread::spawn(move || s.put("x", "1"))
+ };
+ paused_rx.recv().unwrap();
+ let c = {
+ let s = Arc::clone(&store);
+ std::thread::spawn(move || s.put("y", "2"))
+ };
+ // Give C time to run into the gap if it can.
+ std::thread::sleep(Duration::from_millis(200));
+ release_tx.send(()).unwrap();
+ a.join().unwrap();
+ c.join().unwrap();
+ assert!(
+ store.len() <= 1,
+ "capacity 1 store holds {} entries",
+ store.len()
+ );
+ assert_eq!(store.get("y").as_deref(), Some("2"));
+ }
+
#[test]
fn store_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}