Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(storage): WAL-shard pinning — acked deletes survive restart

xerj-orgb78624ef1fd5
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index bb4e5e2e1..a187c6a71 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -5842,7 +5842,7 @@ impl Index {
if precomputed_aggs.is_some() {
// Skip the entire segment loop — the fast agg path already
// computed everything we need. `total_count` is set above.
- } else if count_only && shortcut_count.is_some() {
+ } else if count_only && shortcut_count.is_some() && !deletes_present {
// Overwrite `total_count` — the memtable/bounded-collector scan
// above has already looked at the memtable, but we want an
// authoritative per-shape count, not the sum of two sources.
@@ -5854,6 +5854,16 @@ impl Index {
// top (from+size) hit *sources* still get materialised. Their
// `total_count` is overwritten with `shortcut_count` after the
// loop, and the scan runs in bounded (`count_authoritative`) mode.
+ //
+ // `!deletes_present` (2026-07, delete-durability batch): the
+ // shortcut is DELETE-BLIND (physical postings, not live docs).
+ // Pre-fix this size:0 branch took it unconditionally, so a
+ // `{size:0, query:{term:…}}` total counted version-map-deleted
+ // segment-resident docs (live-observed: 50 where truth was 0,
+ // while the same query with size>0 correctly returned 0). Same
+ // correctness-over-speed gate as the size>0 `count_authoritative`
+ // path below; with ghosts present we fall through to the full
+ // delete-aware counting scan.
total_count = shortcut_count.unwrap();
} else if is_match_all && count_only {
// Legacy MatchAll fast path (covers the `try_shortcut_count`
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 79823822e..bac418966 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -348,6 +348,36 @@ pub struct IndexStore {
/// is already unlinked at retire time, so `recover_orphaned_segments`
/// can never resurrect them as duplicates).
retired_segments: Mutex<Vec<SegmentId>>,
+ /// Delete-durability fix (2026-07): `doc_id → (delete seq_no, wal
+ /// shard)` for every acknowledged delete whose ONLY durable record
+ /// is still its `WalEntry::Delete` in the WAL.
+ ///
+ /// Background: a delete is expressed as (a) a WAL entry, (b) an
+ /// in-RAM version-map tombstone, and (c) the FTS/storage memtables
+ /// dropping the doc. Segment flushes carry NO tombstones the
+ /// reopen path can see (`rebuild_version_map_from_segments` loads
+ /// every segment-resident doc as live), so until a background merge
+ /// physically drops the doc from all segments, the WAL entry is the
+ /// only thing standing between an acked delete and resurrection on
+ /// restart. Pre-fix, every flush's WAL maintenance (checkpoint +
+ /// force-rotate + prune) destroyed those entries — `prune()` deletes
+ /// any rotated generation that has a checkpoint file, regardless of
+ /// the checkpoint's `max_seq_no` — so `DELETE → ack → _flush/_refresh
+ /// /shutdown → restart` brought the docs back (batch-5 adversarial
+ /// verifier, 2026-07-09).
+ ///
+ /// Invariant: WAL maintenance MUST NOT checkpoint/rotate/prune a WAL
+ /// shard that appears in this map ("pinned"). Entries are recorded
+ /// BEFORE the WAL append (so a maintenance pass that rechecks under
+ /// the WAL shard mutex can never miss a racing delete) and removed
+ /// by `sweep_pending_deletes` once the delete is subsumed — i.e. the
+ /// doc was re-indexed with a newer seq_no AND that newer version has
+ /// been flushed into a real segment. Deletes that are never
+ /// subsumed pin their WAL shard (bounded retention growth on
+ /// delete-heavy workloads) — the accepted RC trade-off; the durable
+ /// design is segment-level tombstones (see SectionType::Tombstones
+ /// note in the flush path).
+ pending_deletes: Mutex<std::collections::HashMap<String, (SeqNo, usize)>>,
}
const WAL_MAINTENANCE_INTERVAL_MS: u64 = 1_000;
@@ -407,6 +437,7 @@ impl IndexStore {
last_wal_maintenance_ms: AtomicU64::new(0),
read_leases: std::sync::atomic::AtomicUsize::new(0),
retired_segments: Mutex::new(Vec::new()),
+ pending_deletes: Mutex::new(std::collections::HashMap::new()),
});
// Rebuild version map from flushed segments first (so WAL replay can
@@ -977,11 +1008,37 @@ impl IndexStore {
let entry = WalEntry::Delete {
doc_id: doc_id.to_owned(),
};
+ let ws = self.wal_shard_for(doc_id);
+ // Delete-durability: pin this WAL shard BEFORE appending the
+ // Delete entry. WAL maintenance rechecks `pending_deletes`
+ // under the WAL shard mutex, so ordering the map insert before
+ // the append guarantees maintenance can never checkpoint+rotate+
+ // prune a generation containing this Delete: if maintenance
+ // acquired the shard mutex after our append, our insert is
+ // already visible to its recheck. The placeholder seq_no is
+ // fixed up right after the append assigns the real one.
+ self.pending_deletes
+ .lock()
+ .unwrap()
+ .insert(doc_id.to_owned(), (SeqNo::MAX, ws));
let seq_no = {
- let ws = self.wal_shard_for(doc_id);
let mut wal = self.wal_lock_shard(ws);
- wal.append(&entry)?
+ match wal.append(&entry) {
+ Ok(s) => s,
+ Err(e) => {
+ // Nothing reached the WAL — unpin so the shard's
+ // maintenance isn't blocked forever by a failed op.
+ self.pending_deletes.lock().unwrap().remove(doc_id);
+ return Err(e);
+ }
+ }
};
+ if let Some(slot) = self.pending_deletes.lock().unwrap().get_mut(doc_id) {
+ // Keep the larger seq if a concurrent re-delete raced us.
+ if slot.0 == SeqNo::MAX || slot.0 < seq_no {
+ slot.0 = seq_no;
+ }
+ }
self.version_map
.delete(doc_id, seq_no, IN_MEMORY_SEGMENT_ID)?;
@@ -1201,7 +1258,20 @@ impl IndexStore {
writer.add_section(SectionType::Stored, &encoded)?;
}
- // Build tombstone section if any deletes
+ // Build tombstone section if any deletes.
+ //
+ // HONESTY NOTE (2026-07, delete-durability fix): this section is
+ // currently WRITE-ONLY — nothing in the tree reads it back at
+ // reopen (`rebuild_version_map_from_segments` loads every doc
+ // from `.ids` sidecars / stored sections as live and never looks
+ // at SectionType::Tombstones), so writing it does NOT make
+ // deletes durable. Restart durability of acked deletes is
+ // instead guaranteed by WAL-shard pinning (`pending_deletes` —
+ // see `IndexStore::delete` / `sweep_pending_deletes`). Kept for
+ // format stability; the principled follow-up is to write
+ // `(doc_id, seq_no)` tombstones here on the ENGINE flush path
+ // too and apply them seq-aware at reopen, which would replace
+ // the WAL pinning.
let tombstone_ids: Vec<&str> = entries
.iter()
.filter(|e| e.source.is_none())
@@ -1390,8 +1460,19 @@ impl IndexStore {
// mode. Clean shutdown / explicit `_flush` persists via
// `force_wal_maintenance()`.
self.save_snapshot()?;
- for ws in &self.wal_shards {
+ // Delete-durability: unpin deletes subsumed by flushed newer
+ // versions, then skip maintenance on any shard whose WAL is
+ // still the only durable record of an acked delete.
+ self.sweep_pending_deletes();
+ for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
let mut wal = ws.lock().unwrap();
+ if self.wal_shard_pinned_by_pending_delete(ws_idx) {
+ debug!(
+ shard = ws_idx,
+ "WAL maintenance skipped: shard pinned by unpersisted delete"
+ );
+ continue;
+ }
wal.checkpoint(max_seq)?;
// Disk-space fix (2026-07): force-rotate + prune instead of
// `rotate_if_large(64 MB)`. The checkpoint we just wrote
@@ -1464,12 +1545,68 @@ impl IndexStore {
self.seq_counter.load(std::sync::atomic::Ordering::Acquire)
}
+ /// Delete-durability: drop `pending_deletes` entries whose delete has
+ /// been SUBSUMED by a newer, segment-durable version of the same doc —
+ /// the version map shows the doc live with a seq_no newer than the
+ /// delete AND pointing at a real segment (the flush repointed it off
+ /// `__memtable__`). Once that newer copy is in a segment, reopen
+ /// rebuilds the doc from it (max-seq-wins) and the old `WalEntry::
+ /// Delete` is no longer load-bearing, so its WAL shard can resume
+ /// checkpoint/rotate/prune.
+ ///
+ /// Deliberately conservative: tombstoned docs stay pinned even after
+ /// a background merge physically drops them from the merged segments,
+ /// because an older copy of the doc may still live in a segment that
+ /// was NOT part of the merge — clearing on "merge purged it" alone
+ /// would resurrect from that older segment. Cost: WAL retention on
+ /// delete-heavy indices (bounded by delete volume, zero on append-only
+ /// workloads). Runs only on the 1s-gated / explicit maintenance
+ /// paths — never on the ingest hot path.
+ fn sweep_pending_deletes(&self) {
+ let mut pending = self.pending_deletes.lock().unwrap();
+ if pending.is_empty() {
+ return;
+ }
+ pending.retain(|doc_id, &mut (del_seq, _)| {
+ match self.version_map.get(doc_id) {
+ // Subsumed: live, strictly newer, and already flushed
+ // into a real segment → safe to unpin.
+ Some(e) if !e.deleted && e.seq_no > del_seq => {
+ &*e.segment_id == IN_MEMORY_SEGMENT_ID
+ }
+ // Tombstoned / missing / older-or-equal seq: the WAL
+ // Delete entry is still the only durable record.
+ _ => true,
+ }
+ });
+ }
+
+ /// Delete-durability: true if WAL shard `shard_idx` still holds an
+ /// unpersisted acked delete and therefore MUST NOT be checkpointed,
+ /// rotated, or pruned (`WalWriter::prune` deletes any rotated
+ /// generation that has a checkpoint file — irrespective of the
+ /// checkpoint's `max_seq_no` — so a checkpoint alone already dooms
+ /// the Delete entries). Callers hold the shard's WAL mutex when
+ /// they consult this, which combined with the insert-before-append
+ /// ordering in [`IndexStore::delete`] makes the check race-free.
+ fn wal_shard_pinned_by_pending_delete(&self, shard_idx: usize) -> bool {
+ let pending = self.pending_deletes.lock().unwrap();
+ pending.values().any(|&(_, ws)| ws == shard_idx)
+ }
+
/// Write a checkpoint to ALL WAL shards with the given max_seq_no.
/// Used by `Index::flush` to write a global checkpoint after
/// multi-shard parallel flush.
pub fn wal_checkpoint_all(&self, max_seq: SeqNo) -> Result<()> {
- for ws in &self.wal_shards {
+ self.sweep_pending_deletes();
+ for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
let mut wal = ws.lock().unwrap();
+ // Delete-durability: a checkpoint alone already marks the
+ // shard's current generation prunable, so pinned shards must
+ // skip it (see `wal_shard_pinned_by_pending_delete`).
+ if self.wal_shard_pinned_by_pending_delete(ws_idx) {
+ continue;
+ }
wal.checkpoint(max_seq)?;
}
Ok(())
@@ -1492,8 +1629,22 @@ impl IndexStore {
// on-disk snapshot covering every segment whose WAL is about to
// be dropped. Mirrors the coupling in the gated flush path.
self.save_snapshot()?;
- for ws in &self.wal_shards {
+ // Delete-durability: same shard-pinning rule as the gated
+ // background path — an explicit `_flush`/`_refresh`/shutdown
+ // flush must not destroy the WAL Delete entries that are the
+ // only durable record of acked deletes (pre-fix this very call,
+ // reached from `Index::flush` — including the SIGTERM final
+ // flush — pruned them and deleted docs resurrected on restart).
+ self.sweep_pending_deletes();
+ for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
let mut wal = ws.lock().unwrap();
+ if self.wal_shard_pinned_by_pending_delete(ws_idx) {
+ debug!(
+ shard = ws_idx,
+ "forced WAL maintenance skipped: shard pinned by unpersisted delete"
+ );
+ continue;
+ }
wal.checkpoint(max_seq)?;
wal.force_rotate()?;
wal.prune()?;
@@ -1845,9 +1996,30 @@ impl IndexStore {
}
WalEntry::Delete { doc_id } => {
let seq_no = replay_entry.seq_no;
- self.version_map
+ let applied = self
+ .version_map
.delete(&doc_id, seq_no, IN_MEMORY_SEGMENT_ID)
- .ok();
+ .unwrap_or(false);
+ // Delete-durability: re-pin the WAL shard whenever the
+ // tombstone applied to a doc that still exists (it may
+ // be live in a segment) — this Delete entry remains the
+ // only durable record of the delete, so maintenance
+ // after THIS restart must keep refusing to prune it;
+ // otherwise the delete survives one restart and the doc
+ // resurrects on the next. A delete that applied to
+ // nothing (doc already merge-purged from every segment,
+ // or already tombstoned by an earlier pinned entry) is
+ // vacuous and must not pin the shard forever.
+ // Superseded pins (doc re-indexed later in the replay
+ // stream) are cleared by the next
+ // `sweep_pending_deletes` once the newer version is
+ // segment-resident.
+ if applied {
+ self.pending_deletes
+ .lock()
+ .unwrap()
+ .insert(doc_id.clone(), (seq_no, self.wal_shard_for(&doc_id)));
+ }
let shard = self.shard_for(&doc_id);
let mut mem = self.memtable_shards[shard].lock().unwrap();
mem.push(MemEntry {