AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(storage): reclaim retained disk — WAL rotate+prune on background flush, delete merged-away segments at commit, real seq_nos in merge version map
xerj-orgafc812dddd76
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 2fa900488..eeeb2acd0 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -2025,6 +2025,12 @@ impl Index {Vec<xerj_storage::segment::SegmentId>,xerj_storage::segment::SegmentMeta,usize,+ // (seq_no, doc_id) per surviving doc, in stored order — used to+ // write the merge output's `.ids` side-car after `apply_merge`+ // commits (flush already writes one; merge outputs previously+ // never got one, forcing the slow decode-stored fallback in+ // `rebuild_version_map_from_segments` on every reopen).+ Vec<(u64, String)>,);// Build the list of (batch, metas) pairs we'll launch.@@ -2125,6 +2131,12 @@ impl Index {// Halves merge working memory and frees the Arc<Value>// immediately after both passes complete.let mut fts_input: Vec<(String, HashMap<String, String>, Value)> = Vec::new();+ // (seq_no, doc_id) for every surviving doc — kept exactly+ // aligned with the stored-section byte copy (pushed right+ // after `live_doc_count += 1`, BEFORE the per-doc Value+ // parse that can `continue`), so the `.ids` side-car covers+ // all stored docs even if a doc fails the FTS/DV re-parse.+ let mut ids_pairs: Vec<(u64, String)> = Vec::new();for meta in &metas_for_task {min_seq = min_seq.min(meta.min_seq_no);@@ -2187,6 +2199,7 @@ impl Index {first_doc = false;merged_json_buf.extend_from_slice(raw_str.as_bytes());live_doc_count += 1;+ ids_pairs.push((id_seq.seq_no, id_seq.id.to_string()));// Per-doc Value parse for FTS / DV builders.let doc_value: Value = match serde_json::from_str(raw_str) {@@ -2258,23 +2271,36 @@ impl Index {}// Update version_map so doc → segment_id points to the- // merged segment. We use the (id, source) data already- // collected during the byte-copy pass above; this used to- // re-walk the entire `merged_docs` Vec a second time.+ // merged segment, using each doc's REAL seq_no from+ // `ids_pairs` (collected during the byte-copy pass).+ //+ // CRITICAL correctness fix (2026-07): the previous code set+ // EVERY surviving doc's entry to the segment's `max_seq`+ // ("monotone upper bound"). That poisoned the version map:+ // on the NEXT merge round the survivor filter compares+ // `entry.seq_no > stored._seq_no` and, with entry.seq_no+ // faked to max_seq, every doc except the one that actually+ // owns max_seq looks stale and is dropped. Any second-+ // level merge (merged segment merged again — routine on a+ // converging index) silently discarded ~all docs; live-+ // verified as 200 k → 12 docs after one background merge+ // cascade. The loss was masked on restart only because+ // merged-away input segments were never deleted from disk+ // and `recover_orphaned_segments` resurrected them — a+ // crutch removed by the merge-commit file deletion below.+ //+ // `set_if_latest` (>= guard) rather than unconditional+ // `set`: a doc updated while the merge ran already has a+ // newer entry that must not be clobbered by the merged+ // (older) copy.{- // We need (id, seq_no) pairs. Stash them while we have- // access to fts_input ids and the per-doc source. The- // seq_no isn't carried in fts_input — pull from the doc- // source's sibling _seq_no by re-parsing once… or just- // use the merged segment's max_seq_no as a monotone- // upper bound for all surviving docs. That matches- // what the pre-M4.8 code did when `_seq_no` was missing- // (`unwrap_or(0)`), and is correct for the version-map- // contract because the surviving doc's seq_no is by- // definition ≤ max_seq.- for (id, _, _) in &fts_input {- store_for_task.version_map.set(- id, max_seq, merged_meta.id.as_str(), false,+ let merged_arc: Arc<str> = Arc::from(merged_meta.id.as_str());+ for (seq_no, id) in &ids_pairs {+ store_for_task.version_map.set_if_latest(+ id.as_str(),+ *seq_no,+ Arc::clone(&merged_arc),+ false,);}}@@ -2296,7 +2322,7 @@ impl Index {}}- Some((batch_for_task, merged_meta, live_doc_count as usize))+ Some((batch_for_task, merged_meta, live_doc_count as usize, ids_pairs))})};@@ -2319,14 +2345,55 @@ impl Index {// order.let handle = in_flight.remove(0);match handle.await {- Ok(Some((batch_slice, merged_meta, live_doc_count))) => {+ Ok(Some((batch_slice, merged_meta, live_doc_count, ids_pairs))) => {if let Err(e) = self.store.apply_merge(&batch_slice, merged_meta.clone()) {tracing::warn!("merge: apply_merge failed: {e}");} else {+ // Write the merge output's `.ids` side-car AFTER the+ // merge is committed: if we crashed before+ // `apply_merge`, an output segment carrying a valid+ // `.ids` would be resurrected by+ // `recover_orphaned_segments` on restart alongside+ // its still-live inputs (duplicate docs). Written+ // here, a crash in between merely loses the side-car+ // and reopen falls back to the (slow but correct)+ // decode-stored path — the pre-fix status quo.+ {+ let pairs: Vec<(u64, &str)> = ids_pairs+ .iter()+ .map(|(seq, id)| (*seq, id.as_str()))+ .collect();+ if let Err(e) = self+ .store+ .write_ids_sidecar(merged_meta.id.as_str(), &pairs)+ {+ tracing::warn!(+ merged_id = merged_meta.id.as_str(),+ "merge: failed to write .ids sidecar: {e}"+ );+ }+ }+ // Disk-space fix (2026-07): the input segments are+ // now unreachable from the (persisted) snapshot —+ // unlink their files immediately instead of leaving+ // them for the next restart's orphan cleanup (~137 MB+ // of dead segments after the 1 M-doc benchmark).+ // Drop the cached SegmentReader mmaps first so the+ // kernel can actually free the unlinked blocks once+ // in-flight readers finish (in-flight queries that+ // already hold an Arc<SegmentReader> keep working —+ // unlink under a live mmap is safe on Linux).+ for id in &batch_slice {+ self.store.evict_segment_reader_cache(id.as_str());+ }+ let (rm_files, rm_bytes) =+ self.store.delete_segment_files(&batch_slice);tracing::info!(merged_id = merged_meta.id.as_str(),inputs = batch_slice.len(),live_docs = live_doc_count,+ removed_files = rm_files,+ removed_mb = rm_bytes / 1_000_000,"segment merge complete");// These input segment ids are now unreachable —diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 4780b68ba..4869adacf 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -578,6 +578,90 @@ impl IndexStore {Ok(())}+ /// Write the `<segment_id>.ids` side-car from `(seq_no, doc_id)` pairs+ /// (ZID2 format — see the format comment at the flush-time call in+ /// `finalize_flush_with_publisher`). Shared by the flush path and the+ /// engine merge task: pre-2026-07 only flush wrote the side-car, so+ /// merge-output segments always fell back to the slow decode-stored+ /// path in `rebuild_version_map_from_segments` on reopen (the very+ /// path the side-car exists to avoid — ~302 s vs ~5 s cold restart on+ /// the 66.5 M-doc workload).+ pub fn write_ids_sidecar(+ &self,+ segment_id: &str,+ pairs: &[(u64, &str)],+ ) -> std::io::Result<()> {+ let mut body: Vec<u8> =+ Vec::with_capacity(pairs.iter().map(|(_, id)| 8 + 2 + id.len()).sum::<usize>());+ for (seq_no, id) in pairs {+ body.extend_from_slice(&seq_no.to_le_bytes());+ body.extend_from_slice(&(id.len() as u16).to_le_bytes());+ body.extend_from_slice(id.as_bytes());+ }+ let compressed = lz4_flex::compress_prepend_size(&body);+ let mut buf: Vec<u8> = Vec::with_capacity(8 + compressed.len());+ buf.extend_from_slice(b"ZID2");+ buf.extend_from_slice(&(pairs.len() as u32).to_le_bytes());+ buf.extend_from_slice(&compressed);+ let ids_path = self+ .data_dir+ .join("segments")+ .join(format!("{segment_id}.ids"));+ std::fs::write(&ids_path, &buf)+ }++ /// Unlink every on-disk file belonging to the given segment ids — the+ /// primary `.seg` plus all side-cars (`.sidx`, `.ids`, `.dv`,+ /// `.<field>.post` / `.fst` / `.meta` / `.norms`).+ ///+ /// Disk-space fix (2026-07): called by the engine merge task right+ /// after `apply_merge` commits, so merged-away input segments are+ /// reclaimed immediately instead of lingering until the next process+ /// restart (`cleanup_orphaned_segment_files` only runs on `open()`;+ /// on the 1 M-doc benchmark that left ~137 MB of dead segment files+ /// on disk). Deleting them at commit time also prevents+ /// `recover_orphaned_segments` from resurrecting stale pre-merge+ /// segments (they still carry a valid `.ids` side-car) on restart.+ ///+ /// Unlinking under a live mmap is safe on Linux: snapshot readers+ /// that already opened the segment keep their mappings; the blocks+ /// are freed once the last reader drops. Errors are best-effort —+ /// anything left behind is picked up by the on-open cleanup.+ ///+ /// Returns `(files_removed, bytes_removed)` for logging.+ pub fn delete_segment_files(&self, segment_ids: &[SegmentId]) -> (usize, u64) {+ let segments_dir = self.data_dir.join("segments");+ let ids: std::collections::HashSet<&str> =+ segment_ids.iter().map(|s| s.as_str()).collect();+ if ids.is_empty() {+ return (0, 0);+ }+ let entries = match std::fs::read_dir(&segments_dir) {+ Ok(e) => e,+ Err(_) => return (0, 0),+ };+ let mut removed_files = 0usize;+ let mut removed_bytes = 0u64;+ for entry in entries.flatten() {+ let name = entry.file_name();+ let name_str = name.to_string_lossy();+ // Segment filenames look like "<36-char UUID>.<suffix>".+ if name_str.len() < 37 {+ continue;+ }+ let prefix = &name_str[..36];+ if !ids.contains(prefix) {+ continue;+ }+ let sz = entry.metadata().map(|m| m.len()).unwrap_or(0);+ if std::fs::remove_file(entry.path()).is_ok() {+ removed_files += 1;+ removed_bytes += sz;+ }+ }+ (removed_files, removed_bytes)+ }+// ── Shard routing ─────────────────────────────────────────────────────────/// Route a doc_id to its memtable shard using the *runtime* shard@@ -934,24 +1018,16 @@ impl IndexStore {// ids LZ4 still gets ~2× because the u64 seq_nos step// monotonically. Reading V1 still works for old data dirs.{- let segments_dir = self.data_dir.join("segments");- let ids_path = segments_dir.join(format!("{}.ids", meta.id.as_str()));- let live: Vec<&MemEntry> = entries.iter().filter(|e| e.source.is_some()).collect();- let mut body: Vec<u8> = Vec::with_capacity(- live.iter().map(|e| 8 + 2 + e.doc_id.len()).sum::<usize>(),- );- for e in &live {- body.extend_from_slice(&e.seq_no.to_le_bytes());- body.extend_from_slice(&(e.doc_id.len() as u16).to_le_bytes());- body.extend_from_slice(e.doc_id.as_bytes());- }- let compressed = lz4_flex::compress_prepend_size(&body);- let mut buf: Vec<u8> = Vec::with_capacity(4 + 4 + compressed.len());- buf.extend_from_slice(b"ZID2");- buf.extend_from_slice(&(live.len() as u32).to_le_bytes());- buf.extend_from_slice(&compressed);- if let Err(e) = std::fs::write(&ids_path, &buf) {- tracing::warn!(?ids_path, "failed to write seg.ids sidecar: {e}");+ let pairs: Vec<(u64, &str)> = entries+ .iter()+ .filter(|e| e.source.is_some())+ .map(|e| (e.seq_no, e.doc_id.as_str()))+ .collect();+ if let Err(e) = self.write_ids_sidecar(meta.id.as_str(), &pairs) {+ tracing::warn!(+ segment_id = meta.id.as_str(),+ "failed to write seg.ids sidecar: {e}"+ );}}@@ -1042,7 +1118,23 @@ impl IndexStore {for ws in &self.wal_shards {let mut wal = ws.lock().unwrap();wal.checkpoint(max_seq)?;- let _ = wal.rotate_if_large(64 * 1024 * 1024);+ // Disk-space fix (2026-07): force-rotate + prune instead of+ // `rotate_if_large(64 MB)`. The checkpoint we just wrote+ // records the shard's FULL current file length (we hold the+ // shard mutex, so `offset == current_offset`), i.e. the+ // whole active generation is covered — rotating now lets+ // `prune()` reclaim it. The old 64 MB threshold never+ // fired on realistic workloads: with 16 WAL shards a 1 M-doc+ // ingest leaves 10-35 MB per shard file, so ~490 MB of WAL+ // that was already durable in segments sat on disk forever+ // (the explicit `_flush` path has done force-rotate via+ // `force_wal_maintenance` since the 2026-04-25 head-to-head;+ // this mirrors it on the background flush path). Rotation+ // churn stays bounded by the surrounding 1 s+ // `WAL_MAINTENANCE_INTERVAL_MS` gate — at most one+ // rotation per shard per second, and `force_rotate` no-ops+ // on generations with no data past the header.+ wal.force_rotate()?;wal.prune()?;}}diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rsindex 023ee129b..ee1e03e7d 100644--- a/engine/crates/xerj-storage/src/version_map.rs+++ b/engine/crates/xerj-storage/src/version_map.rs@@ -106,6 +106,42 @@ impl VersionMap {self.inner.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });}+ /// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the+ /// currently recorded seq_no (or the doc is unknown).+ ///+ /// Used by the merge task to repoint surviving docs at the merged+ /// segment with their REAL per-doc seq_nos:+ /// - `>=` (not `>`): the surviving doc's entry currently points at a+ /// merged-away INPUT segment with the SAME seq_no — equality must+ /// win or `apply_merge → remove_segment` would drop the entry (and+ /// the doc) entirely.+ /// - guarded (not unconditional `set`): a doc updated concurrently+ /// while the merge ran has a NEWER entry pointing at the memtable /+ /// another segment — clobbering it with the merged copy would+ /// resurrect the stale version.+ ///+ /// Atomic per key via the dashmap entry API.+ pub fn set_if_latest(+ &self,+ doc_id: impl Into<String>,+ seq_no: SeqNo,+ segment_id: impl Into<Arc<str>>,+ deleted: bool,+ ) {+ let doc_id = doc_id.into();+ use dashmap::mapref::entry::Entry;+ match self.inner.entry(doc_id) {+ Entry::Occupied(mut occ) => {+ if seq_no >= occ.get().seq_no {+ occ.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ }+ }+ Entry::Vacant(vac) => {+ vac.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });+ }+ }+ }+/// Attempt an atomic compare-and-set.////// - If `expected_seq_no` is `None`, succeeds only if the document does NOT