AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(storage): merge-race undercount — read-lease segment snapshots, graveyard-deferred file deletion
xerj-orgd4b17caa1ec9
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 76c351c4a..1decac35b 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2446,9 +2446,9 @@ impl Index {}// Disk-space fix (2026-07): the input segments are// now unreachable from the (persisted) snapshot —- // unlink their files immediately instead of leaving- // them for the next restart's orphan cleanup (~137 MB- // of dead segments after the 1 M-doc benchmark).+ // reclaim their files instead of leaving them for+ // the next restart's orphan cleanup (~137 MB of+ // dead segments after the 1 M-doc benchmark).// Drop the cached SegmentReader mmaps first so the// kernel can actually free the unlinked blocks once// in-flight readers finish (in-flight queries that@@ -2457,14 +2457,27 @@ impl Index {for id in &batch_slice {self.store.evict_segment_reader_cache(id.as_str());}+ // Merge-race fix (2026-07, follow-up to the disk+ // fix): RETIRE, don't delete. An immediate unlink+ // raced in-flight searches that had snapshotted the+ // pre-merge segment list but not yet opened every+ // segment — the scan silently skipped the vanished+ // segment and undercounted hits.total (observed+ // live: 798,281 instead of 932,037).+ // `retire_segment_files` deletes right away when no+ // snapshot read lease is outstanding, otherwise it+ // parks the ids in a graveyard swept by the last+ // lease drop — so deferral is bounded by the+ // longest-running in-flight query, not a restart.let (rm_files, rm_bytes) =- self.store.delete_segment_files(&batch_slice);+ self.store.retire_segment_files(&batch_slice);tracing::info!(merged_id = merged_meta.id.as_str(),inputs = batch_slice.len(),live_docs = live_doc_count,removed_files = rm_files,removed_mb = rm_bytes / 1_000_000,+ deferred = rm_files == 0,"segment merge complete");// These input segment ids are now unreachable —@@ -4537,7 +4550,14 @@ impl Index {&& !seg_hits.is_empty()&& all_hits.len() < materialisation_limit{- if let Ok(seg_reader) = self.store.open_segment_arc(&seg_id) {+ // Merge-race hardening (2026-07): NEVER+ // silently skip a snapshot segment that+ // fails to open — under the read-lease fix+ // its files are guaranteed present, so an+ // open failure is real corruption and must+ // fail the query, not shrink its results.+ {+ let seg_reader = self.store.open_segment_arc(&seg_id)?;if let Ok(Some(stored_bytes_raw)) =seg_reader.section(SectionType::Stored){@@ -4740,7 +4760,15 @@ impl Index {&mut total_count,);}- } else if let Ok(seg_reader) = self.store.open_segment_arc(&seg_id) {+ } else {+ // Merge-race hardening (2026-07): a snapshot+ // segment that fails to open is a query error, not+ // a segment to skip — skipping silently undercounts+ // hits.total (the observed 798,281-of-932,037 bug).+ // With the read lease held by `snap`, retired+ // segment files persist, so this only fires on+ // genuine corruption.+ let seg_reader = self.store.open_segment_arc(&seg_id)?;if let Ok(Some(stored_bytes_raw)) = seg_reader.section(SectionType::Stored) {if let Ok(stored_bytes) =xerj_storage::stored_codec::decode_stored(stored_bytes_raw)@@ -5255,10 +5283,12 @@ impl Index {.collect();let snap_bg = self.store.snapshot();for seg in &snap_bg.segments {- let reader = match self.store.open_segment_arc(&seg.id) {- Ok(r) => r,- Err(_) => continue,- };+ // Merge-race hardening (2026-07): open failures are+ // errors, not skips — a skipped segment silently drops+ // its docs from every aggregation bucket. The read+ // lease held by `snap_bg` keeps retired segment files+ // on disk, so this only fires on genuine corruption.+ let reader = self.store.open_segment_arc(&seg.id)?;let stored_bytes_raw = match reader.section(SectionType::Stored) {Ok(Some(b)) => b,_ => continue,diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 40c56e87a..687e9fd8c 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -101,6 +101,46 @@ pub struct DrainedMemtable {// ── Fsck report types ─────────────────────────────────────────────────────────//+// ── SnapshotReadGuard ─────────────────────────────────────────────────────────++/// A loaded [`IndexSnapshot`] plus a **read lease** on its segment files.+///+/// Returned by [`IndexStore::snapshot`]. While any guard is alive, files+/// of segments retired by a concurrent merge are parked in a graveyard+/// instead of being unlinked (`IndexStore::retire_segment_files`), so a+/// scan iterating this snapshot's segment list can always open every+/// segment it references. Dropping the last guard sweeps the graveyard.+///+/// Derefs to `Arc<IndexSnapshot>` exactly like the+/// `arc_swap::Guard<Arc<IndexSnapshot>>` it wraps, so call sites are+/// source-compatible with the pre-lease `snapshot()`.+pub struct SnapshotReadGuard<'a> {+ snap: arc_swap::Guard<Arc<IndexSnapshot>>,+ store: &'a IndexStore,+}++impl std::ops::Deref for SnapshotReadGuard<'_> {+ type Target = Arc<IndexSnapshot>;+ fn deref(&self) -> &Arc<IndexSnapshot> {+ &self.snap+ }+}++impl Drop for SnapshotReadGuard<'_> {+ fn drop(&mut self) {+ // Last lease out sweeps the graveyard. `fetch_sub` returning 1+ // means this was the final outstanding lease.+ if self+ .store+ .read_leases+ .fetch_sub(1, std::sync::atomic::Ordering::SeqCst)+ == 1+ {+ self.store.sweep_retired_segments();+ }+ }+}+// Returned by `IndexStore::fsck_segments()`. Per-section CRC32C is// computed at write time and validated on every section_checked()// call. The fast `section()` read path skips revalidation for perf;@@ -275,6 +315,21 @@ pub struct IndexStore {/// every `WAL_MAINTENANCE_INTERVAL_MS` runs it on behalf of all/// concurrent flushers.last_wal_maintenance_ms: AtomicU64,+ /// Merge-race fix (2026-07) — number of outstanding+ /// [`SnapshotReadGuard`]s (read leases). Every reader that obtains a+ /// segment list via [`IndexStore::snapshot`] holds a lease for as long+ /// as it keeps the guard, and retired (merged-away) segment files are+ /// only unlinked once this count reaches zero. See+ /// `retire_segment_files` for the full race description.+ read_leases: std::sync::atomic::AtomicUsize,+ /// Graveyard of segment ids retired by `apply_merge` whose on-disk+ /// files could not be deleted immediately because a read lease was+ /// outstanding. Swept (files unlinked, reader-cache entries evicted)+ /// by the last lease drop; crash leftovers are handled by the on-open+ /// `cleanup_orphaned_segment_files` (their `.ids` resurrection marker+ /// is already unlinked at retire time, so `recover_orphaned_segments`+ /// can never resurrect them as duplicates).+ retired_segments: Mutex<Vec<SegmentId>>,}const WAL_MAINTENANCE_INTERVAL_MS: u64 = 1_000;@@ -328,6 +383,8 @@ impl IndexStore {memtable_bytes: AtomicU64::new(0),seg_reader_cache: dashmap::DashMap::new(),last_wal_maintenance_ms: AtomicU64::new(0),+ read_leases: std::sync::atomic::AtomicUsize::new(0),+ retired_segments: Mutex::new(Vec::new()),});// Rebuild version map from flushed segments first (so WAL replay can@@ -662,6 +719,85 @@ impl IndexStore {(removed_files, removed_bytes)}+ /// Retire merged-away segments: delete their files **as soon as it is+ /// safe**, i.e. once no in-flight reader can still be holding a+ /// pre-merge segment list that references them.+ ///+ /// Merge-race fix (2026-07): `run_merge_once` used to call+ /// [`delete_segment_files`](Self::delete_segment_files) directly after+ /// `apply_merge`. A search that had already loaded the pre-merge+ /// snapshot would then hit `open_segment_arc` for a segment whose+ /// files had just been unlinked, get an error, and SILENTLY SKIP the+ /// segment — returning an undercounted `hits.total` (observed live:+ /// 798,281 instead of 932,037). Now every reader holds a+ /// [`SnapshotReadGuard`] lease; if any lease is outstanding the ids+ /// are parked in `retired_segments` and swept by the last lease drop.+ ///+ /// The `.ids` side-car is unlinked IMMEDIATELY regardless of leases:+ /// it is only read at open/recovery time (never on the query path),+ /// and removing it up front keeps the disk-fix invariant that a crash+ /// while deletions are deferred cannot let+ /// `recover_orphaned_segments` resurrect the merged-away inputs as+ /// duplicates on restart (recovery requires a valid `.ids`). Any+ /// other leftover files are removed by the on-open+ /// `cleanup_orphaned_segment_files`.+ ///+ /// Returns `(files_removed, bytes_removed)` — `(0, 0)` when deletion+ /// was deferred to the graveyard.+ pub fn retire_segment_files(&self, segment_ids: &[SegmentId]) -> (usize, u64) {+ if segment_ids.is_empty() {+ return (0, 0);+ }+ // Kill the resurrection marker first (crash safety, see above).+ let segments_dir = self.data_dir.join("segments");+ for id in segment_ids {+ let _ = std::fs::remove_file(segments_dir.join(format!("{}.ids", id.as_str())));+ }+ {+ let mut graveyard = self.retired_segments.lock().unwrap();+ graveyard.extend_from_slice(segment_ids);+ }+ // Opportunistic sweep: deletes right away when no reader is active+ // (the common case), otherwise the last lease drop sweeps.+ self.sweep_retired_segments()+ }++ /// Delete the files of every graveyard segment, provided no read+ /// lease is outstanding. Called by [`retire_segment_files`] and by+ /// the last [`SnapshotReadGuard`] drop.+ ///+ /// The lease check happens while holding the graveyard lock, and+ /// `snapshot()` increments the lease count with a SeqCst RMW *before*+ /// loading the snapshot pointer (itself stored by `apply_merge`+ /// before retire). So if this observes `read_leases == 0`, any+ /// reader that appears afterwards is guaranteed to load the+ /// post-merge snapshot and can never reference the ids being swept.+ fn sweep_retired_segments(&self) -> (usize, u64) {+ let ids: Vec<SegmentId> = {+ let mut graveyard = self.retired_segments.lock().unwrap();+ if graveyard.is_empty()+ || self.read_leases.load(std::sync::atomic::Ordering::SeqCst) != 0+ {+ return (0, 0);+ }+ graveyard.drain(..).collect()+ };+ // Evict any reader-cache entries a leased scan may have re-opened+ // for these ids so their mmaps (and the unlinked blocks) get+ // released.+ for id in &ids {+ self.seg_reader_cache.remove(id.as_str());+ }+ let (files, bytes) = self.delete_segment_files(&ids);+ debug!(+ segments = ids.len(),+ removed_files = files,+ removed_bytes = bytes,+ "retired segment files swept"+ );+ (files, bytes)+ }+// ── Shard routing ─────────────────────────────────────────────────────────/// Route a doc_id to its memtable shard using the *runtime* shard@@ -1197,8 +1333,30 @@ impl IndexStore {// ── Read path ─────────────────────────────────────────────────────────────/// Load the current snapshot. Lock-free.- pub fn snapshot(&self) -> arc_swap::Guard<Arc<IndexSnapshot>> {- self.snapshot.load()+ ///+ /// Merge-race fix (2026-07): the returned guard is also a **read+ /// lease** — for as long as it is alive, the on-disk files of every+ /// segment it references are guaranteed to exist, even if a+ /// concurrent merge commits and retires some of them+ /// (`retire_segment_files` defers the unlink until the last lease+ /// drops). Pre-fix, `run_merge_once` unlinked merged-away segment+ /// files immediately after `apply_merge`; a search that had already+ /// snapshotted the old segment list would then fail to open those+ /// segments mid-scan and silently skip them (observed live: 798,281+ /// hits returned instead of 932,037 during a background merge).+ ///+ /// IMPORTANT ordering: the lease count is incremented *before* the+ /// snapshot pointer is loaded (`fetch_add` is a full RMW barrier), so+ /// a retire that observes `read_leases == 0` can only race with a+ /// reader that will observe the *post-merge* snapshot — never one+ /// still holding the merged-away segment list.+ pub fn snapshot(&self) -> SnapshotReadGuard<'_> {+ self.read_leases+ .fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ SnapshotReadGuard {+ snap: self.snapshot.load(),+ store: self,+ }}/// Return the current WAL sequence number (the next value that@@ -1336,21 +1494,40 @@ impl IndexStore {return Ok(Arc::clone(entry.value()));}let snap = self.snapshot.load();- let meta = snap- .segments- .iter()- .find(|s| s.id == segment_id)- .ok_or_else(|| StorageError::SegmentNotFound(segment_id.to_owned()))?;+ // Merge-race fix (2026-07): a miss against the CURRENT snapshot no+ // longer means "gone". The caller may hold a `SnapshotReadGuard`+ // on an OLDER snapshot whose segment was merged away after the+ // caller loaded it — its files are then still on disk (retire+ // defers deletion until the last read lease drops), just no longer+ // registered. Fall back to the id-derived filename (`{id}.seg` —+ // the invariant name set by SegmentWriter) so an in-flight scan+ // stays consistent with ITS snapshot instead of silently skipping+ // the segment (the merge-race undercount bug). For a genuinely+ // unknown id the open below fails and the error propagates.+ let seg_path: String = match snap.segments.iter().find(|s| s.id == segment_id) {+ Some(m) => m.seg_path.clone(),+ None => {+ let fallback = format!("{segment_id}.seg");+ let is_local = matches!(self.config.storage_mode, StorageMode::Local);+ if is_local+ && !self.data_dir.join("segments").join(&fallback).exists()+ {+ return Err(StorageError::SegmentNotFound(segment_id.to_owned()));+ }+ fallback+ }+ };+ drop(snap);- let local_path = self.data_dir.join("segments").join(&meta.seg_path);+ let local_path = self.data_dir.join("segments").join(&seg_path);// For object-store mode: check local cache; fetch from backend on miss.let reader = if let StorageMode::ObjectStore { backend, cache_dir } = &self.config.storage_mode {- let cache_path = cache_dir.join(&meta.seg_path);+ let cache_path = cache_dir.join(&seg_path);if cache_path.exists() {crate::segment::SegmentReader::open(cache_path)?} else {- let object_key = format!("segments/{}", meta.seg_path);+ let object_key = format!("segments/{seg_path}");let backend_clone = std::sync::Arc::clone(backend);let key_clone = object_key.clone();let data = tokio::task::block_in_place(|| {@@ -2217,4 +2394,132 @@ mod tests {let reader2 = store.open_segment(&meta.id).unwrap();assert_eq!(reader2.header().doc_count, 1);}++ // ── Merge-race read-lease tests (2026-07) ────────────────────────────────++ /// Build two flushed segments and merge them, returning+ /// (store, input_ids, merged_meta). The merge is applied+ /// (snapshot swapped) but the input files are NOT yet retired.+ fn two_segments_merged(+ dir: &Path,+ ) -> (Arc<IndexStore>, Vec<SegmentId>, SegmentMeta) {+ let store = open_test_store(dir);+ store.index("doc-1", serde_json::json!({"v": 1})).unwrap();+ store.flush().unwrap();+ store.index("doc-2", serde_json::json!({"v": 2})).unwrap();+ store.flush().unwrap();+ let ids: Vec<SegmentId> =+ store.snapshot().segments.iter().map(|s| s.id.clone()).collect();+ assert_eq!(ids.len(), 2);+ let executor = crate::merge::MergeExecutor::new(+ Arc::clone(&store),+ crate::merge::MergeConfig { io_rate_mb_per_sec: 0, ..Default::default() },+ );+ let merged = executor.execute_merge(&ids).unwrap();+ (store, ids, merged)+ }++ #[test]+ fn retire_without_lease_deletes_immediately() {+ let dir = tempfile::tempdir().unwrap();+ let (store, ids, merged) = two_segments_merged(dir.path());+ let segments_dir = dir.path().join("segments");++ for id in &ids {+ assert!(segments_dir.join(format!("{id}.seg")).exists());+ }+ let (files, _bytes) = store.retire_segment_files(&ids);+ assert!(files >= 2, "expected immediate deletion, removed {files} files");+ for id in &ids {+ assert!(+ !segments_dir.join(format!("{id}.seg")).exists(),+ "input segment file should be gone with no lease outstanding"+ );+ }+ assert!(segments_dir.join(format!("{}.seg", merged.id)).exists());+ }++ #[test]+ fn retire_defers_deletion_while_lease_held_and_scan_stays_consistent() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ store.index("doc-1", serde_json::json!({"v": 1})).unwrap();+ store.flush().unwrap();+ store.index("doc-2", serde_json::json!({"v": 2})).unwrap();+ store.flush().unwrap();++ // A "query" snapshots the segment list BEFORE the merge commits.+ let query_snap = store.snapshot();+ let ids: Vec<SegmentId> =+ query_snap.segments.iter().map(|s| s.id.clone()).collect();+ assert_eq!(ids.len(), 2);++ // Merge commits and retires the inputs while the query is in flight+ // (mirrors run_merge_once: evict reader cache, then retire).+ let executor = crate::merge::MergeExecutor::new(+ Arc::clone(&store),+ crate::merge::MergeConfig { io_rate_mb_per_sec: 0, ..Default::default() },+ );+ executor.execute_merge(&ids).unwrap();+ for id in &ids {+ store.evict_segment_reader_cache(id.as_str());+ }+ let (files, _bytes) = store.retire_segment_files(&ids);+ assert_eq!(files, 0, "deletion must be deferred while a lease is held");++ let segments_dir = dir.path().join("segments");+ for id in &ids {+ assert!(+ segments_dir.join(format!("{id}.seg")).exists(),+ "retired segment file must survive until the last lease drops"+ );+ assert!(+ !segments_dir.join(format!("{id}.ids")).exists(),+ ".ids resurrection marker must be unlinked at retire time"+ );+ // The in-flight query can still open every segment of ITS+ // snapshot (fallback open path — the ids are no longer in the+ // current snapshot and the reader cache was evicted).+ let reader = store.open_segment_arc(id.as_str()).unwrap();+ assert_eq!(reader.header().doc_count, 1);+ }++ // Query finishes → last lease drops → graveyard swept.+ drop(query_snap);+ for id in &ids {+ assert!(+ !segments_dir.join(format!("{id}.seg")).exists(),+ "retired segment file must be deleted once the last lease drops"+ );+ }+ }++ #[test]+ fn crash_with_deferred_retire_does_not_resurrect_inputs() {+ let dir = tempfile::tempdir().unwrap();+ let (store, ids, merged) = two_segments_merged(dir.path());++ // Hold a lease so retire defers, then "crash" (leak the lease and+ // drop the store) — the input .seg files stay on disk, .ids gone.+ let leaked = store.snapshot();+ let (files, _bytes) = store.retire_segment_files(&ids);+ assert_eq!(files, 0);+ std::mem::forget(leaked);+ drop(store);++ // Reopen: recover_orphaned_segments must NOT resurrect the+ // merged-away inputs (no .ids side-car), and the on-open cleanup+ // must reclaim their leftover files.+ let store2 = open_test_store(dir.path());+ let snap = store2.snapshot();+ assert_eq!(snap.segments.len(), 1, "only the merged segment survives");+ assert_eq!(snap.segments[0].id, merged.id);+ let segments_dir = dir.path().join("segments");+ for id in &ids {+ assert!(+ !segments_dir.join(format!("{id}.seg")).exists(),+ "crash leftovers must be cleaned on open"+ );+ }+ }}