Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(storage): reclaim retained disk — WAL rotate+prune on background flush, delete merged-away segments at commit, real seq_nos in merge version map

xerj-orgafc812dddd76
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 2fa900488..eeeb2acd0 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2025,6 +2025,12 @@ impl Index {
Vec<xerj_storage::segment::SegmentId>,
xerj_storage::segment::SegmentMeta,
usize,
+ // (seq_no, doc_id) per surviving doc, in stored order — used to
+ // write the merge output's `.ids` side-car after `apply_merge`
+ // commits (flush already writes one; merge outputs previously
+ // never got one, forcing the slow decode-stored fallback in
+ // `rebuild_version_map_from_segments` on every reopen).
+ Vec<(u64, String)>,
);
// Build the list of (batch, metas) pairs we'll launch.
@@ -2125,6 +2131,12 @@ impl Index {
// Halves merge working memory and frees the Arc<Value>
// immediately after both passes complete.
let mut fts_input: Vec<(String, HashMap<String, String>, Value)> = Vec::new();
+ // (seq_no, doc_id) for every surviving doc — kept exactly
+ // aligned with the stored-section byte copy (pushed right
+ // after `live_doc_count += 1`, BEFORE the per-doc Value
+ // parse that can `continue`), so the `.ids` side-car covers
+ // all stored docs even if a doc fails the FTS/DV re-parse.
+ let mut ids_pairs: Vec<(u64, String)> = Vec::new();
for meta in &metas_for_task {
min_seq = min_seq.min(meta.min_seq_no);
@@ -2187,6 +2199,7 @@ impl Index {
first_doc = false;
merged_json_buf.extend_from_slice(raw_str.as_bytes());
live_doc_count += 1;
+ ids_pairs.push((id_seq.seq_no, id_seq.id.to_string()));
// Per-doc Value parse for FTS / DV builders.
let doc_value: Value = match serde_json::from_str(raw_str) {
@@ -2258,23 +2271,36 @@ impl Index {
}
// Update version_map so doc → segment_id points to the
- // merged segment. We use the (id, source) data already
- // collected during the byte-copy pass above; this used to
- // re-walk the entire `merged_docs` Vec a second time.
+ // merged segment, using each doc's REAL seq_no from
+ // `ids_pairs` (collected during the byte-copy pass).
+ //
+ // CRITICAL correctness fix (2026-07): the previous code set
+ // EVERY surviving doc's entry to the segment's `max_seq`
+ // ("monotone upper bound"). That poisoned the version map:
+ // on the NEXT merge round the survivor filter compares
+ // `entry.seq_no > stored._seq_no` and, with entry.seq_no
+ // faked to max_seq, every doc except the one that actually
+ // owns max_seq looks stale and is dropped. Any second-
+ // level merge (merged segment merged again — routine on a
+ // converging index) silently discarded ~all docs; live-
+ // verified as 200 k → 12 docs after one background merge
+ // cascade. The loss was masked on restart only because
+ // merged-away input segments were never deleted from disk
+ // and `recover_orphaned_segments` resurrected them — a
+ // crutch removed by the merge-commit file deletion below.
+ //
+ // `set_if_latest` (>= guard) rather than unconditional
+ // `set`: a doc updated while the merge ran already has a
+ // newer entry that must not be clobbered by the merged
+ // (older) copy.
{
- // We need (id, seq_no) pairs. Stash them while we have
- // access to fts_input ids and the per-doc source. The
- // seq_no isn't carried in fts_input — pull from the doc
- // source's sibling _seq_no by re-parsing once… or just
- // use the merged segment's max_seq_no as a monotone
- // upper bound for all surviving docs. That matches
- // what the pre-M4.8 code did when `_seq_no` was missing
- // (`unwrap_or(0)`), and is correct for the version-map
- // contract because the surviving doc's seq_no is by
- // definition ≤ max_seq.
- for (id, _, _) in &fts_input {
- store_for_task.version_map.set(
- id, max_seq, merged_meta.id.as_str(), false,
+ let merged_arc: Arc<str> = Arc::from(merged_meta.id.as_str());
+ for (seq_no, id) in &ids_pairs {
+ store_for_task.version_map.set_if_latest(
+ id.as_str(),
+ *seq_no,
+ Arc::clone(&merged_arc),
+ false,
);
}
}
@@ -2296,7 +2322,7 @@ impl Index {
}
}
- Some((batch_for_task, merged_meta, live_doc_count as usize))
+ Some((batch_for_task, merged_meta, live_doc_count as usize, ids_pairs))
})
};
@@ -2319,14 +2345,55 @@ impl Index {
// order.
let handle = in_flight.remove(0);
match handle.await {
- Ok(Some((batch_slice, merged_meta, live_doc_count))) => {
+ Ok(Some((batch_slice, merged_meta, live_doc_count, ids_pairs))) => {
if let Err(e) = self.store.apply_merge(&batch_slice, merged_meta.clone()) {
tracing::warn!("merge: apply_merge failed: {e}");
} else {
+ // Write the merge output's `.ids` side-car AFTER the
+ // merge is committed: if we crashed before
+ // `apply_merge`, an output segment carrying a valid
+ // `.ids` would be resurrected by
+ // `recover_orphaned_segments` on restart alongside
+ // its still-live inputs (duplicate docs). Written
+ // here, a crash in between merely loses the side-car
+ // and reopen falls back to the (slow but correct)
+ // decode-stored path — the pre-fix status quo.
+ {
+ let pairs: Vec<(u64, &str)> = ids_pairs
+ .iter()
+ .map(|(seq, id)| (*seq, id.as_str()))
+ .collect();
+ if let Err(e) = self
+ .store
+ .write_ids_sidecar(merged_meta.id.as_str(), &pairs)
+ {
+ tracing::warn!(
+ merged_id = merged_meta.id.as_str(),
+ "merge: failed to write .ids sidecar: {e}"
+ );
+ }
+ }
+ // Disk-space fix (2026-07): the input segments are
+ // now unreachable from the (persisted) snapshot —
+ // unlink their files immediately instead of leaving
+ // them for the next restart's orphan cleanup (~137 MB
+ // of dead segments after the 1 M-doc benchmark).
+ // Drop the cached SegmentReader mmaps first so the
+ // kernel can actually free the unlinked blocks once
+ // in-flight readers finish (in-flight queries that
+ // already hold an Arc<SegmentReader> keep working —
+ // unlink under a live mmap is safe on Linux).
+ for id in &batch_slice {
+ self.store.evict_segment_reader_cache(id.as_str());
+ }
+ let (rm_files, rm_bytes) =
+ self.store.delete_segment_files(&batch_slice);
tracing::info!(
merged_id = merged_meta.id.as_str(),
inputs = batch_slice.len(),
live_docs = live_doc_count,
+ removed_files = rm_files,
+ removed_mb = rm_bytes / 1_000_000,
"segment merge complete"
);
// These input segment ids are now unreachable —
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 4780b68ba..4869adacf 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -578,6 +578,90 @@ impl IndexStore {
Ok(())
}
+ /// Write the `<segment_id>.ids` side-car from `(seq_no, doc_id)` pairs
+ /// (ZID2 format — see the format comment at the flush-time call in
+ /// `finalize_flush_with_publisher`). Shared by the flush path and the
+ /// engine merge task: pre-2026-07 only flush wrote the side-car, so
+ /// merge-output segments always fell back to the slow decode-stored
+ /// path in `rebuild_version_map_from_segments` on reopen (the very
+ /// path the side-car exists to avoid — ~302 s vs ~5 s cold restart on
+ /// the 66.5 M-doc workload).
+ pub fn write_ids_sidecar(
+ &self,
+ segment_id: &str,
+ pairs: &[(u64, &str)],
+ ) -> std::io::Result<()> {
+ let mut body: Vec<u8> =
+ Vec::with_capacity(pairs.iter().map(|(_, id)| 8 + 2 + id.len()).sum::<usize>());
+ for (seq_no, id) in pairs {
+ body.extend_from_slice(&seq_no.to_le_bytes());
+ body.extend_from_slice(&(id.len() as u16).to_le_bytes());
+ body.extend_from_slice(id.as_bytes());
+ }
+ let compressed = lz4_flex::compress_prepend_size(&body);
+ let mut buf: Vec<u8> = Vec::with_capacity(8 + compressed.len());
+ buf.extend_from_slice(b"ZID2");
+ buf.extend_from_slice(&(pairs.len() as u32).to_le_bytes());
+ buf.extend_from_slice(&compressed);
+ let ids_path = self
+ .data_dir
+ .join("segments")
+ .join(format!("{segment_id}.ids"));
+ std::fs::write(&ids_path, &buf)
+ }
+
+ /// Unlink every on-disk file belonging to the given segment ids — the
+ /// primary `.seg` plus all side-cars (`.sidx`, `.ids`, `.dv`,
+ /// `.<field>.post` / `.fst` / `.meta` / `.norms`).
+ ///
+ /// Disk-space fix (2026-07): called by the engine merge task right
+ /// after `apply_merge` commits, so merged-away input segments are
+ /// reclaimed immediately instead of lingering until the next process
+ /// restart (`cleanup_orphaned_segment_files` only runs on `open()`;
+ /// on the 1 M-doc benchmark that left ~137 MB of dead segment files
+ /// on disk). Deleting them at commit time also prevents
+ /// `recover_orphaned_segments` from resurrecting stale pre-merge
+ /// segments (they still carry a valid `.ids` side-car) on restart.
+ ///
+ /// Unlinking under a live mmap is safe on Linux: snapshot readers
+ /// that already opened the segment keep their mappings; the blocks
+ /// are freed once the last reader drops. Errors are best-effort —
+ /// anything left behind is picked up by the on-open cleanup.
+ ///
+ /// Returns `(files_removed, bytes_removed)` for logging.
+ pub fn delete_segment_files(&self, segment_ids: &[SegmentId]) -> (usize, u64) {
+ let segments_dir = self.data_dir.join("segments");
+ let ids: std::collections::HashSet<&str> =
+ segment_ids.iter().map(|s| s.as_str()).collect();
+ if ids.is_empty() {
+ return (0, 0);
+ }
+ let entries = match std::fs::read_dir(&segments_dir) {
+ Ok(e) => e,
+ Err(_) => return (0, 0),
+ };
+ let mut removed_files = 0usize;
+ let mut removed_bytes = 0u64;
+ for entry in entries.flatten() {
+ let name = entry.file_name();
+ let name_str = name.to_string_lossy();
+ // Segment filenames look like "<36-char UUID>.<suffix>".
+ if name_str.len() < 37 {
+ continue;
+ }
+ let prefix = &name_str[..36];
+ if !ids.contains(prefix) {
+ continue;
+ }
+ let sz = entry.metadata().map(|m| m.len()).unwrap_or(0);
+ if std::fs::remove_file(entry.path()).is_ok() {
+ removed_files += 1;
+ removed_bytes += sz;
+ }
+ }
+ (removed_files, removed_bytes)
+ }
+
// ── Shard routing ─────────────────────────────────────────────────────────
/// Route a doc_id to its memtable shard using the *runtime* shard
@@ -934,24 +1018,16 @@ impl IndexStore {
// ids LZ4 still gets ~2× because the u64 seq_nos step
// monotonically. Reading V1 still works for old data dirs.
{
- let segments_dir = self.data_dir.join("segments");
- let ids_path = segments_dir.join(format!("{}.ids", meta.id.as_str()));
- let live: Vec<&MemEntry> = entries.iter().filter(|e| e.source.is_some()).collect();
- let mut body: Vec<u8> = Vec::with_capacity(
- live.iter().map(|e| 8 + 2 + e.doc_id.len()).sum::<usize>(),
- );
- for e in &live {
- body.extend_from_slice(&e.seq_no.to_le_bytes());
- body.extend_from_slice(&(e.doc_id.len() as u16).to_le_bytes());
- body.extend_from_slice(e.doc_id.as_bytes());
- }
- let compressed = lz4_flex::compress_prepend_size(&body);
- let mut buf: Vec<u8> = Vec::with_capacity(4 + 4 + compressed.len());
- buf.extend_from_slice(b"ZID2");
- buf.extend_from_slice(&(live.len() as u32).to_le_bytes());
- buf.extend_from_slice(&compressed);
- if let Err(e) = std::fs::write(&ids_path, &buf) {
- tracing::warn!(?ids_path, "failed to write seg.ids sidecar: {e}");
+ let pairs: Vec<(u64, &str)> = entries
+ .iter()
+ .filter(|e| e.source.is_some())
+ .map(|e| (e.seq_no, e.doc_id.as_str()))
+ .collect();
+ if let Err(e) = self.write_ids_sidecar(meta.id.as_str(), &pairs) {
+ tracing::warn!(
+ segment_id = meta.id.as_str(),
+ "failed to write seg.ids sidecar: {e}"
+ );
}
}
@@ -1042,7 +1118,23 @@ impl IndexStore {
for ws in &self.wal_shards {
let mut wal = ws.lock().unwrap();
wal.checkpoint(max_seq)?;
- let _ = wal.rotate_if_large(64 * 1024 * 1024);
+ // Disk-space fix (2026-07): force-rotate + prune instead of
+ // `rotate_if_large(64 MB)`. The checkpoint we just wrote
+ // records the shard's FULL current file length (we hold the
+ // shard mutex, so `offset == current_offset`), i.e. the
+ // whole active generation is covered — rotating now lets
+ // `prune()` reclaim it. The old 64 MB threshold never
+ // fired on realistic workloads: with 16 WAL shards a 1 M-doc
+ // ingest leaves 10-35 MB per shard file, so ~490 MB of WAL
+ // that was already durable in segments sat on disk forever
+ // (the explicit `_flush` path has done force-rotate via
+ // `force_wal_maintenance` since the 2026-04-25 head-to-head;
+ // this mirrors it on the background flush path). Rotation
+ // churn stays bounded by the surrounding 1 s
+ // `WAL_MAINTENANCE_INTERVAL_MS` gate — at most one
+ // rotation per shard per second, and `force_rotate` no-ops
+ // on generations with no data past the header.
+ wal.force_rotate()?;
wal.prune()?;
}
}
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 023ee129b..ee1e03e7d 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -106,6 +106,42 @@ impl VersionMap {
self.inner.insert(doc_id, VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
}
+ /// Record `doc_id → (seq_no, segment_id)` only if `seq_no` is >= the
+ /// currently recorded seq_no (or the doc is unknown).
+ ///
+ /// Used by the merge task to repoint surviving docs at the merged
+ /// segment with their REAL per-doc seq_nos:
+ /// - `>=` (not `>`): the surviving doc's entry currently points at a
+ /// merged-away INPUT segment with the SAME seq_no — equality must
+ /// win or `apply_merge → remove_segment` would drop the entry (and
+ /// the doc) entirely.
+ /// - guarded (not unconditional `set`): a doc updated concurrently
+ /// while the merge ran has a NEWER entry pointing at the memtable /
+ /// another segment — clobbering it with the merged copy would
+ /// resurrect the stale version.
+ ///
+ /// Atomic per key via the dashmap entry API.
+ pub fn set_if_latest(
+ &self,
+ doc_id: impl Into<String>,
+ seq_no: SeqNo,
+ segment_id: impl Into<Arc<str>>,
+ deleted: bool,
+ ) {
+ let doc_id = doc_id.into();
+ use dashmap::mapref::entry::Entry;
+ match self.inner.entry(doc_id) {
+ Entry::Occupied(mut occ) => {
+ if seq_no >= occ.get().seq_no {
+ occ.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ }
+ }
+ Entry::Vacant(vac) => {
+ vac.insert(VersionEntry { seq_no, segment_id: segment_id.into(), deleted });
+ }
+ }
+ }
+
/// Attempt an atomic compare-and-set.
///
/// - If `expected_seq_no` is `None`, succeeds only if the document does NOT