Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(storage,engine): segment-durable delete tombstones end WAL pinning; GET never 404s in the merge-publish window (RC4 W2 #14 + #15)

xerj-org17c2c3628bcf
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 160363ed3..6a10b2c77 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -2872,6 +2872,15 @@ impl Index {
// from the M5.22 profile.
let mut survivors: Vec<(u64, String, String)> = Vec::new();
+ // RC4 W2 #14 — union of the inputs' SEQ-AWARE tombstone
+ // sections (ZTB2), max seq per doc. A load-bearing
+ // tombstone (doc deleted, an OLDER copy possibly live in
+ // a segment OUTSIDE this batch) must survive the merge:
+ // its input segment is removed by `apply_merge`, so
+ // without carry-forward the next restart's rebuild would
+ // resurrect the older copy.
+ let mut tomb_union: HashMap<String, u64> = HashMap::new();
+
for meta in &metas_for_task {
min_seq = min_seq.min(meta.min_seq_no);
max_seq = max_seq.max(meta.max_seq_no);
@@ -2895,6 +2904,55 @@ impl Index {
return None;
}
};
+ // Collect tombstones BEFORE the stored-section read:
+ // tombstone-only segments (doc_count 0) `continue`
+ // out of the doc pass below but their pairs must
+ // still ride along. A tombstone section that fails
+ // to read/decode aborts the batch like any other
+ // unreadable input (dropping it silently would
+ // resurrect deleted docs after the inputs are gone).
+ if meta.has_tombstones {
+ match reader.section(SectionType::Tombstones) {
+ Ok(Some(tb)) => {
+ match xerj_storage::segment::decode_tombstones_v2(tb) {
+ Some(pairs) => {
+ for (seq, id) in pairs {
+ let e = tomb_union.entry(id).or_insert(0);
+ if seq > *e {
+ *e = seq;
+ }
+ }
+ }
+ None if tb.starts_with(b"ZTB2") => {
+ // Corrupt V2 payload — dropping it
+ // silently would resurrect deleted
+ // docs once the input is removed.
+ tracing::error!(
+ ?seg_path,
+ "merge ABORTED: corrupt ZTB2 tombstone \
+ section (inputs preserved)"
+ );
+ failed_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
+ // Legacy id-only JSON payload: no seq
+ // info — those deletes are WAL-pinned,
+ // not segment-durable; nothing to carry.
+ None => {}
+ }
+ }
+ Ok(None) => {}
+ Err(e) => {
+ tracing::error!(
+ ?seg_path,
+ "merge ABORTED: failed to read tombstone section \
+ (inputs preserved): {e}"
+ );
+ failed_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
+ }
+ }
let stored_bytes_raw = match reader.section(SectionType::Stored) {
Ok(Some(b)) => b,
Ok(None) => {
@@ -3061,6 +3119,45 @@ impl Index {
return None;
}
+ // RC4 W2 #14 — carry still-load-bearing tombstones into
+ // the merged output. GC rule (mirrors the
+ // `sweep_pending_deletes` unpin proof): a tombstone
+ // subsumed by a strictly newer, SEGMENT-durable live
+ // version can be dropped — every rebuild would ignore it
+ // anyway (max-seq-wins). Everything else is carried:
+ // an older copy of the doc may live in a segment outside
+ // this batch, and the tombstone is what keeps it dead
+ // across restarts.
+ let tomb_carry: Vec<(u64, String)> = tomb_union
+ .into_iter()
+ .filter(|(id, seq)| match store_for_task.version_map.get(id) {
+ Some(e)
+ if !e.deleted
+ && e.seq_no > *seq
+ && &*e.segment_id
+ != xerj_storage::version_map::IN_MEMORY_SEGMENT_ID =>
+ {
+ false
+ }
+ _ => true,
+ })
+ .map(|(id, seq)| (seq, id))
+ .collect();
+ if !tomb_carry.is_empty() {
+ let pair_refs: Vec<(u64, &str)> = tomb_carry
+ .iter()
+ .map(|(seq, id)| (*seq, id.as_str()))
+ .collect();
+ if let Err(e) = writer.add_section(
+ SectionType::Tombstones,
+ xerj_storage::segment::encode_tombstones_v2(&pair_refs),
+ ) {
+ tracing::error!("merge ABORTED: failed to add tombstone section: {e}");
+ failed_for_task.fetch_add(1, Ordering::Relaxed);
+ return None;
+ }
+ }
+
let merged_meta = match writer.finish(live_doc_count, min_seq, max_seq) {
Ok(m) => m,
Err(e) => {
@@ -3133,6 +3230,21 @@ impl Index {
false,
);
}
+ // RC4 W2 #14 — repoint carried tombstones onto the
+ // merged segment BEFORE `apply_merge` runs: its
+ // `remove_segment` purges every entry still pointing
+ // at a merged-away input, and a purged tombstone
+ // would resurrect the doc in live search. Guarded
+ // (`set_if_latest`): a doc re-indexed while the
+ // merge ran keeps its newer live entry.
+ for (seq, id) in &tomb_carry {
+ store_for_task.version_map.set_if_latest(
+ id.as_str(),
+ *seq,
+ Arc::clone(&merged_arc),
+ true,
+ );
+ }
}
// Doc-values side-car — reuse the same `Value`s we
@@ -5003,10 +5115,33 @@ impl Index {
if let Some(ver) = self.store.version_map.get(id) {
let seg_id: std::sync::Arc<str> = std::sync::Arc::clone(&ver.segment_id);
drop(ver);
- if let Some(meta) = snap.segments.iter().find(|m| m.id.as_str() == &*seg_id) {
+ // RC4 W2 #15 — resolve the owning segment's doc_count from the
+ // snapshot, FALLING BACK to the on-disk segment header when the
+ // id isn't listed there. During a merge there is a window
+ // between the version-map repoint (`set_if_latest` onto the
+ // merged id, inside the merge task) and the snapshot publish
+ // (`apply_merge`) — pre-fix, a GET in that window found no
+ // snapshot meta, every legacy-loop segment was skipped by the
+ // "not this segment" check below, and an EXISTING doc returned
+ // 404 (live repro: 47 transient 404s hammering 8 ids through
+ // one forcemerge). The merged segment file is fully durable
+ // before the repoint (`SegmentWriter::finish` fsyncs), and
+ // `open_segment_arc` derives `{id}.seg` for ids missing from
+ // the snapshot, so opening it directly is race-free.
+ let seg_doc_count: Option<u64> =
+ match snap.segments.iter().find(|m| m.id.as_str() == &*seg_id) {
+ Some(meta) => Some(meta.doc_count),
+ None if &*seg_id != xerj_storage::version_map::IN_MEMORY_SEGMENT_ID => self
+ .store
+ .open_segment_arc(&seg_id)
+ .ok()
+ .map(|r| r.header().doc_count),
+ None => None,
+ };
+ if let Some(doc_count) = seg_doc_count {
if let (Some(pos_map), Some(slices)) = (
- self.id_pos_map_for(&seg_id, meta.doc_count),
- self.stored_slices_for(&seg_id, meta.doc_count),
+ self.id_pos_map_for(&seg_id, doc_count),
+ self.stored_slices_for(&seg_id, doc_count),
) {
if let Some(&pos) = pos_map.get(id) {
if let Some(&(start, end)) = slices.offsets.get(pos as usize) {
@@ -5053,6 +5188,36 @@ impl Index {
}
}
}
+
+ // RC4 W2 #15 — last-resort direct open for the merge-publish
+ // window: the version map points at a segment the snapshot does
+ // not list (merged output between repoint and `apply_merge`) and
+ // the slice-cache fast path above couldn't serve it (e.g. caches
+ // not yet warm). Decode that one segment's stored section
+ // directly rather than 404-ing an existing doc.
+ if let Some(ver) = self.store.version_map.get(id) {
+ let seg_id: std::sync::Arc<str> = std::sync::Arc::clone(&ver.segment_id);
+ drop(ver);
+ if &*seg_id != xerj_storage::version_map::IN_MEMORY_SEGMENT_ID
+ && !snap.segments.iter().any(|m| m.id.as_str() == &*seg_id)
+ {
+ if let Ok(reader) = self.store.open_segment_arc(&seg_id) {
+ if let Ok(Some(stored_bytes_raw)) = reader.section(SectionType::Stored) {
+ if let Ok(stored_bytes) =
+ xerj_storage::stored_codec::decode_stored(stored_bytes_raw)
+ {
+ if let Ok(docs) = serde_json::from_slice::<Vec<Value>>(&stored_bytes) {
+ for doc in docs {
+ if doc.get("_id").and_then(Value::as_str) == Some(id) {
+ return Ok(doc.get("_source").cloned());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
drop(snap);
Ok(None)
diff --git a/engine/crates/xerj-engine/tests/rc4_w2_storage_hardening.rs b/engine/crates/xerj-engine/tests/rc4_w2_storage_hardening.rs
new file mode 100644
index 000000000..99faf47bf
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/rc4_w2_storage_hardening.rs
@@ -0,0 +1,172 @@
+//! RC4 Wave-2 Stream D — storage/durability hardening regressions.
+//!
+//! Item 15: transient GET 404 under merge (version map repointed to the
+//! merged segment before the snapshot publish).
+//! Item 14: delete tombstones must survive merges (carry-forward) now that
+//! the WAL pin is released once a tombstone is segment-resident.
+
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::Arc;
+
+use serde_json::json;
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+use xerj_engine::Engine;
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+/// Item 15 — GET must never 404 an existing doc while a forcemerge
+/// publishes. Live repro pre-fix: 47/15072 GETs returned 404 in the
+/// ~300 ms window between the merge task's version-map repoint and
+/// `apply_merge`'s snapshot swap (8×25k docs, HTTP). In-process the
+/// hammer is much faster than HTTP, so even the smaller window here
+/// catches the regression reliably.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn get_document_never_404s_during_forcemerge() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ engine.create_index("m15", Schema::empty()).unwrap();
+ let idx = engine.get_index("m15").unwrap();
+
+ // 6 flushed batches → 6+ segments, one probe id per batch.
+ let mut probe_ids = Vec::new();
+ for b in 0..6 {
+ for i in 0..1500 {
+ let id = format!("d{b}_{i}");
+ idx.index_document(
+ Some(id),
+ json!({"title": format!("doc {b}/{i}"), "body": "lorem ipsum dolor sit amet", "n": b * 1500 + i}),
+ )
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+ probe_ids.push(format!("d{b}_7"));
+ }
+
+ // Sanity: every probe resolves before the merge.
+ for id in &probe_ids {
+ assert!(
+ idx.get_document(id).await.unwrap().is_some(),
+ "probe {id} must exist pre-merge"
+ );
+ }
+
+ // Hammer GETs concurrently with the forcemerge.
+ let stop = Arc::new(AtomicBool::new(false));
+ let misses = Arc::new(AtomicU64::new(0));
+ let total = Arc::new(AtomicU64::new(0));
+ let hammer = {
+ let idx = Arc::clone(&idx);
+ let stop = Arc::clone(&stop);
+ let misses = Arc::clone(&misses);
+ let total = Arc::clone(&total);
+ let ids = probe_ids.clone();
+ tokio::spawn(async move {
+ while !stop.load(Ordering::Relaxed) {
+ for id in &ids {
+ match idx.get_document(id).await {
+ Ok(Some(_)) => {}
+ Ok(None) => {
+ misses.fetch_add(1, Ordering::Relaxed);
+ }
+ Err(_) => {
+ misses.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ total.fetch_add(1, Ordering::Relaxed);
+ }
+ tokio::task::yield_now().await;
+ }
+ })
+ };
+
+ let merged = idx.force_merge(1).await.unwrap();
+ assert!(merged > 0, "forcemerge must merge something");
+ // Let the hammer overlap any post-merge publish work too.
+ tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+ stop.store(true, Ordering::Relaxed);
+ hammer.await.unwrap();
+
+ let t = total.load(Ordering::Relaxed);
+ let m = misses.load(Ordering::Relaxed);
+ assert!(t > 0, "hammer must have run");
+ assert_eq!(
+ m, 0,
+ "GET returned 404/error for existing docs during forcemerge ({m}/{t})"
+ );
+
+ // And everything still resolves after.
+ for id in &probe_ids {
+ assert!(
+ idx.get_document(id).await.unwrap().is_some(),
+ "probe {id} lost after merge"
+ );
+ }
+}
+
+/// Item 14 — with WAL pins released once tombstones are segment-resident,
+/// a merge must CARRY the tombstones forward: the deleted doc stays dead
+/// in live search AND across a restart, even after every pre-merge
+/// segment (including the tombstone-only one) has been merged away and
+/// the delete's WAL entry pruned.
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+async fn deleted_doc_stays_dead_through_merge_and_restart() {
+ let dir = TempDir::new().unwrap();
+ {
+ let engine = make_engine(&dir);
+ engine.create_index("m14", Schema::empty()).unwrap();
+ let idx = engine.get_index("m14").unwrap();
+
+ for b in 0..3 {
+ for i in 0..50 {
+ idx.index_document(Some(format!("k{b}_{i}")), json!({"v": i, "b": b}))
+ .await
+ .unwrap();
+ }
+ idx.flush().await.unwrap();
+ }
+
+ // Plain delete, never re-indexed.
+ assert!(idx.delete_document("k0_7").await.unwrap());
+ // Flush → maintenance persists the tombstone (tombstone-only
+ // segment) and releases the WAL pin.
+ idx.flush().await.unwrap();
+ assert!(idx.get_document("k0_7").await.unwrap().is_none());
+
+ // Merge everything — inputs (including the tombstone-only
+ // segment) are removed; the tombstone must ride along.
+ idx.force_merge(1).await.unwrap();
+ assert!(
+ idx.get_document("k0_7").await.unwrap().is_none(),
+ "deleted doc resurrected in live search after merge"
+ );
+ assert!(
+ idx.get_document("k0_8").await.unwrap().is_some(),
+ "sibling doc lost after merge"
+ );
+ idx.flush().await.unwrap();
+ drop(idx);
+ drop(engine);
+ }
+
+ // Restart — the WAL delete entry is pruned; only the merged
+ // segment's carried tombstone can keep the doc dead.
+ let engine2 = make_engine(&dir);
+ let idx2 = engine2.get_index("m14").unwrap();
+ assert!(
+ idx2.get_document("k0_7").await.unwrap().is_none(),
+ "deleted doc resurrected after merge + restart (tombstone not carried forward)"
+ );
+ for probe in ["k0_8", "k1_7", "k2_49"] {
+ assert!(
+ idx2.get_document(probe).await.unwrap().is_some(),
+ "live doc {probe} lost after merge + restart"
+ );
+ }
+}
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 20f7e6a56..f5a37b683 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -676,6 +676,37 @@ impl IndexStore {
let seg_path = segments_dir.join(&seg_filename);
let ids_path = segments_dir.join(format!("{prefix}.ids"));
+ // RC4 W2 #14 — TOMBSTONE-ONLY orphan (doc_count 0, ZTB2
+ // section, no `.ids` — written by `persist_pending_tombstones`
+ // whose crash-window is prune-before-snapshot-save). The
+ // `.seg` file itself is the durability marker: `finish()`
+ // fsyncs file + dir before anything can reference it, and
+ // `SegmentReader::open` re-validates the whole-file CRC here.
+ // Without resurrection the on-open cleanup would delete it —
+ // and with the delete's WAL entry already pruned, the doc
+ // would come back from the dead.
+ if let Ok(reader) = SegmentReader::open(&seg_path) {
+ let hdr = reader.header();
+ if hdr.doc_count == 0 && (hdr.flags & 0x0001) != 0 {
+ let seg_meta = match std::fs::metadata(&seg_path) {
+ Ok(m) => m,
+ Err(_) => continue,
+ };
+ recovered.push(SegmentMeta {
+ id: prefix.to_string(),
+ doc_count: 0,
+ size_bytes: seg_meta.len(),
+ min_seq_no: hdr.min_seq_no,
+ max_seq_no: hdr.max_seq_no,
+ created_at_ms: hdr.created_at_ms,
+ has_tombstones: true,
+ seg_path: seg_filename,
+ sidx_path: format!("{prefix}.sidx"),
+ });
+ continue;
+ }
+ }
+
// .ids sidecar must exist — it's the last write of the flush
// sequence, so its presence implies the segment is complete.
let ids_bytes = match std::fs::read(&ids_path) {
@@ -726,10 +757,15 @@ impl IndexStore {
continue;
}
- // Sanity-check the segment file itself opens.
- if SegmentReader::open(&seg_path).is_err() {
- continue;
- }
+ // Sanity-check the segment file itself opens, and preserve
+ // its tombstone flag (RC4 W2 #14 — a resurrected
+ // delete-carrying segment must keep `has_tombstones` so the
+ // rebuild applies its ZTB2 pairs and the deletes_present
+ // gates stay on).
+ let has_tombstones = match SegmentReader::open(&seg_path) {
+ Ok(r) => (r.header().flags & 0x0001) != 0,
+ Err(_) => continue,
+ };
let seg_meta = match std::fs::metadata(&seg_path) {
Ok(m) => m,
@@ -749,7 +785,7 @@ impl IndexStore {
min_seq_no: min_seq,
max_seq_no: max_seq,
created_at_ms,
- has_tombstones: false,
+ has_tombstones,
seg_path: seg_filename,
sidx_path: format!("{prefix}.sidx"),
});
@@ -1392,27 +1428,26 @@ impl IndexStore {
writer.add_section(SectionType::Stored, &encoded)?;
}
- // Build tombstone section if any deletes.
- //
- // HONESTY NOTE (2026-07, delete-durability fix): this section is
- // currently WRITE-ONLY — nothing in the tree reads it back at
- // reopen (`rebuild_version_map_from_segments` loads every doc
- // from `.ids` sidecars / stored sections as live and never looks
- // at SectionType::Tombstones), so writing it does NOT make
- // deletes durable. Restart durability of acked deletes is
- // instead guaranteed by WAL-shard pinning (`pending_deletes` —
- // see `IndexStore::delete` / `sweep_pending_deletes`). Kept for
- // format stability; the principled follow-up is to write
- // `(doc_id, seq_no)` tombstones here on the ENGINE flush path
- // too and apply them seq-aware at reopen, which would replace
- // the WAL pinning.
- let tombstone_ids: Vec<&str> = entries
+ // Build the SEQ-AWARE tombstone section (ZTB2) for any drained
+ // deletes (RC4 W2 #14). Reopen now APPLIES these pairs
+ // (`rebuild_version_map_from_segments`, max-seq-wins), which makes
+ // the delete segment-durable and lets `sweep_pending_deletes`
+ // unpin the WAL shard once this segment lands — pre-fix the
+ // pinning was FOREVER for a plain never-re-indexed delete (live
+ // repro: one DELETE → WAL grew 866 KB → 3 MB across 6 flushed
+ // rounds and was never pruned again). The engine flush path
+ // drains its own memtable (no delete entries here); its deletes
+ // are persisted by `persist_pending_tombstones` on the
+ // maintenance tick instead. The pre-fix payload (JSON id array,
+ // write-only, no seqs) is dropped; ZTB2 is self-describing and
+ // old sections are ignored by the decoder.
+ let tombstone_pairs: Vec<(u64, &str)> = entries
.iter()
.filter(|e| e.source.is_none())
- .map(|e| e.doc_id.as_str())
+ .map(|e| (e.seq_no, e.doc_id.as_str()))
.collect();
- if !tombstone_ids.is_empty() {
- let ts_bytes = serde_json::to_vec(&tombstone_ids)?;
+ if !tombstone_pairs.is_empty() {
+ let ts_bytes = crate::segment::encode_tombstones_v2(&tombstone_pairs);
writer.add_section(SectionType::Tombstones, &ts_bytes)?;
}
@@ -1513,6 +1548,23 @@ impl IndexStore {
std::sync::Arc::clone(&segment_id_arc),
false,
);
+ } else {
+ // RC4 W2 #14 — repoint the delete's tombstone off
+ // `__memtable__` onto the segment that now durably
+ // carries it (the ZTB2 section written above; the
+ // segment + section were fsynced by `writer.finish`).
+ // Guarded (`set_if_latest`): a doc re-indexed while this
+ // flush ran has a strictly newer live entry that must
+ // not be clobbered back to deleted. Once
+ // segment-resident, `sweep_pending_deletes` unpins the
+ // WAL shard and `wal_pair_durable` lets the Delete
+ // entry prune.
+ self.version_map.set_if_latest(
+ &entry.doc_id,
+ entry.seq_no,
+ std::sync::Arc::clone(&segment_id_arc),
+ true,
+ );
}
}
if prof {
@@ -1593,6 +1645,14 @@ impl IndexStore {
// widens that already-handled window; it adds no new failure
// mode. Clean shutdown / explicit `_flush` persists via
// `force_wal_maintenance()`.
+ // RC4 W2 #14 — persist still-memtable-resident acked deletes
+ // as a tombstone-only segment BEFORE the snapshot save, so
+ // the save registers it and the prune below can release the
+ // deletes' WAL pins. Best-effort: on failure the deletes
+ // simply stay WAL-pinned (retention, not loss).
+ if let Err(e) = self.persist_pending_tombstones() {
+ warn!(error = %e, "tombstone persistence failed — deletes stay WAL-pinned");
+ }
self.save_snapshot()?;
// RC4 W1 #8 — verified maintenance (see
// `wal_maintain_all_verified`). The pre-fix loop here
@@ -1693,13 +1753,99 @@ impl IndexStore {
Some(e) if !e.deleted && e.seq_no > del_seq => {
&*e.segment_id == IN_MEMORY_SEGMENT_ID
}
- // Tombstoned / missing / older-or-equal seq: the WAL
- // Delete entry is still the only durable record.
+ // RC4 W2 #14 — tombstone persisted: the delete (or an
+ // equal-or-newer one) is recorded in a segment's ZTB2
+ // tombstone section (the repoint off `__memtable__`
+ // happens only after the segment is durably on disk),
+ // so reopen reconstructs it without the WAL entry →
+ // unpin. Pre-fix, a plain never-re-indexed delete
+ // matched the `_ => true` arm FOREVER and its WAL shard
+ // never checkpointed/rotated/pruned again.
+ Some(e) if e.deleted && e.seq_no >= del_seq => {
+ &*e.segment_id == IN_MEMORY_SEGMENT_ID
+ }
+ // Missing / older live seq: the WAL Delete entry is
+ // still the only durable record.
_ => true,
}
});
}
+ /// RC4 W2 #14 — make every still-memtable-resident acked delete
+ /// segment-durable by writing a dedicated TOMBSTONE-ONLY segment
+ /// (doc_count = 0, single ZTB2 `Tombstones` section).
+ ///
+ /// The engine flush path drains its own memtable, so plain deletes
+ /// (which live only in `pending_deletes` + the version map) never
+ /// reach `finalize_flush_with_publisher`'s tombstone section. This
+ /// runs on the WAL-maintenance paths (1 s gated tick + explicit
+ /// `force_wal_maintenance`) right before the snapshot is persisted:
+ /// collect → write segment (fsynced by `SegmentWriter::finish`) →
+ /// repoint the version-map tombstones onto it → publish to the
+ /// snapshot. The subsequent `save_snapshot()` + verified prune then
+ /// see a segment-resident tombstone and release the WAL pin.
+ ///
+ /// Crash-window: a crash after the prune but before `save_snapshot`
+ /// is covered by `recover_orphaned_segments`, which resurrects
+ /// orphan tombstone-only segments from their (CRC-validated) header.
+ fn persist_pending_tombstones(&self) -> Result<()> {
+ // Collect deletes whose tombstone is still memtable-resident.
+ let pairs: Vec<(u64, String)> = {
+ let pending = self.pending_deletes.lock().unwrap();
+ if pending.is_empty() {
+ return Ok(());
+ }
+ pending
+ .keys()
+ .filter_map(|doc_id| match self.version_map.get(doc_id) {
+ Some(e) if e.deleted && &*e.segment_id == IN_MEMORY_SEGMENT_ID => {
+ Some((e.seq_no, doc_id.clone()))
+ }
+ _ => None,
+ })
+ .collect()
+ };
+ if pairs.is_empty() {
+ return Ok(());
+ }
+
+ let min_seq = pairs.iter().map(|(s, _)| *s).min().unwrap_or(0);
+ let max_seq = pairs.iter().map(|(s, _)| *s).max().unwrap_or(0);
+ let pair_refs: Vec<(u64, &str)> = pairs.iter().map(|(s, id)| (*s, id.as_str())).collect();
+
+ let segments_dir = self.data_dir.join("segments");
+ let mut writer = SegmentWriter::new(&segments_dir, self.config.schema_version, 0, 0)?;
+ writer.add_section(
+ SectionType::Tombstones,
+ crate::segment::encode_tombstones_v2(&pair_refs),
+ )?;
+ // doc_count 0: tombstone-only. finish() fsyncs file + dir.
+ let meta = writer.finish(0, min_seq, max_seq)?;
+
+ // Segment is durable — repoint the tombstones onto it (guarded:
+ // a doc re-indexed since collection keeps its newer live entry).
+ let seg_arc: std::sync::Arc<str> = std::sync::Arc::from(meta.id.as_str());
+ for (seq, doc_id) in &pairs {
+ self.version_map.set_if_latest(
+ doc_id.as_str(),
+ *seq,
+ std::sync::Arc::clone(&seg_arc),
+ true,
+ );
+ }
+
+ // Publish so the caller's save_snapshot() registers it on disk.
+ self.snapshot
+ .rcu(|old| Arc::new(old.with_new_segment(meta.clone())));
+
+ info!(
+ segment_id = meta.id.as_str(),
+ tombstones = pairs.len(),
+ "tombstone-only segment persisted (acked deletes now segment-durable)"
+ );
+ Ok(())
+ }
+
/// Delete-durability: true if WAL shard `shard_idx` still holds an
/// unpersisted acked delete and therefore MUST NOT be checkpointed,
/// rotated, or pruned. Callers hold the shard's WAL mutex when
@@ -1745,17 +1891,19 @@ impl IndexStore {
fn wal_pair_durable(&self, is_delete: bool, doc_id: &str, seq: SeqNo) -> bool {
if is_delete {
match self.version_map.get(doc_id) {
- Some(e) if !e.deleted => {
- e.seq_no > seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID
- }
+ Some(e) if !e.deleted => e.seq_no > seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID,
+ // RC4 W2 #14 — the tombstone itself (this delete or a
+ // newer one) is segment-resident: reopen rebuilds the
+ // deletion from the ZTB2 section, the WAL entry is no
+ // longer load-bearing.
+ Some(e) if e.deleted => e.seq_no >= seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID,
_ => false,
}
} else {
match self.version_map.get(doc_id) {
Some(e) if e.deleted => e.seq_no >= seq,
Some(e) => {
- e.seq_no > seq
- || (e.seq_no == seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID)
+ e.seq_no > seq || (e.seq_no == seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID)
}
None => false,
}
@@ -1895,10 +2043,7 @@ impl IndexStore {
/// Collect the `(is_delete, doc_id, seq)` pairs of every entry NOT yet
/// provable durable (mapping updates are always durable and never
/// collected).
- fn collect_unproven(
- &self,
- entries: &[crate::wal::ReplayEntry],
- ) -> Vec<(bool, String, SeqNo)> {
+ fn collect_unproven(&self, entries: &[crate::wal::ReplayEntry]) -> Vec<(bool, String, SeqNo)> {
entries
.iter()
.filter(|e| !self.wal_entry_durable(&e.entry, e.seq_no))
@@ -1924,6 +2069,11 @@ impl IndexStore {
/// watermark is now computed internally from the persisted snapshot
/// (max seq actually resident in flushed segments).
pub fn force_wal_maintenance(&self) -> Result<()> {
+ // RC4 W2 #14 — see the gated call site: segment-persist acked
+ // deletes so their WAL pins can be released by the prune below.
+ if let Err(e) = self.persist_pending_tombstones() {
+ warn!(error = %e, "tombstone persistence failed — deletes stay WAL-pinned");
+ }
// P2.3 — persist the (possibly debounced) snapshot before pruning
// so an explicit `_flush` / clean shutdown always leaves the
// on-disk snapshot covering every segment whose WAL is about to
@@ -2036,16 +2186,17 @@ impl IndexStore {
// stays consistent with ITS snapshot instead of silently skipping
// the segment (the merge-race undercount bug). For a genuinely
// unknown id the open below fails and the error propagates.
+ // RC4 W2 #15 — no `exists()` pre-check on the fallback path. The
+ // old `exists()`-then-open was a TOCTOU: a retire sweep unlinking
+ // the file between the two calls turned a benign "segment gone"
+ // into a raw io::Error surfacing as a 500 (the mixed-RUW
+ // `store_exception` file race), while an `exists() == false` for
+ // a file mid-rename produced a spurious SegmentNotFound. The
+ // open itself is now the single authority: a NotFound from it is
+ // mapped to `SegmentNotFound` below.
let seg_path: String = match snap.segments.iter().find(|s| s.id == segment_id) {
Some(m) => m.seg_path.clone(),
- None => {
- let fallback = format!("{segment_id}.seg");
- let is_local = matches!(self.config.storage_mode, StorageMode::Local);
- if is_local && !self.data_dir.join("segments").join(&fallback).exists() {
- return Err(StorageError::SegmentNotFound(segment_id.to_owned()));
- }
- fallback
- }
+ None => format!("{segment_id}.seg"),
};
drop(snap);
@@ -2077,7 +2228,16 @@ impl IndexStore {
crate::segment::SegmentReader::open(cache_path)?
}
} else {
- crate::segment::SegmentReader::open(local_path)?
+ match crate::segment::SegmentReader::open(local_path) {
+ Ok(r) => r,
+ // Typed not-found (RC4 W2 #15): the file was already
+ // retired/unlinked — callers treat SegmentNotFound as
+ // "skip / stale snapshot", not as an internal error.
+ Err(StorageError::Io(e)) if e.kind() == std::io::ErrorKind::NotFound => {
+ return Err(StorageError::SegmentNotFound(segment_id.to_owned()));
+ }
+ Err(e) => return Err(e),
+ }
};
let arc = Arc::new(reader);
self.seg_reader_cache
@@ -2261,8 +2421,53 @@ impl IndexStore {
}
}
- if total > 0 {
- info!(total, "version map rebuilt from segments");
+ // RC4 W2 #14 — apply SEGMENT-DURABLE tombstones (ZTB2 sections),
+ // max-seq-wins against the doc entries loaded above. This is
+ // what lets an acked delete survive a restart WITHOUT its WAL
+ // entry (whose retention used to pin the shard's prune forever).
+ // Runs after ALL doc entries are loaded so ordering across
+ // segments cannot matter: a tombstone only lands if no
+ // strictly-newer live version exists anywhere
+ // (`set_if_latest`, and delete-vs-doc seqs are never equal).
+ // Legacy id-only JSON tombstone sections decode to `None` and
+ // are skipped — those deletes are still WAL-pinned as before.
+ let mut tombstones_applied = 0usize;
+ for meta in &snap.segments {
+ if !meta.has_tombstones {
+ continue;
+ }
+ let seg_path = segments_dir.join(&meta.seg_path);
+ let reader = match SegmentReader::open(&seg_path) {
+ Ok(r) => r,
+ Err(e) => {
+ warn!(segment = %meta.id, error = %e, "cannot open segment for tombstone rebuild");
+ continue;
+ }
+ };
+ let ts_bytes = match reader.section(SectionType::Tombstones) {
+ Ok(Some(b)) => b,
+ _ => continue,
+ };
+ let Some(pairs) = crate::segment::decode_tombstones_v2(ts_bytes) else {
+ continue; // legacy id-only payload — no seqs, skip
+ };
+ let seg_id_arc: std::sync::Arc<str> = std::sync::Arc::from(meta.id.as_str());
+ for (seq, doc_id) in pairs {
+ self.version_map.set_if_latest(
+ doc_id,
+ seq,
+ std::sync::Arc::clone(&seg_id_arc),
+ true,
+ );
+ tombstones_applied += 1;
+ }
+ }
+
+ if total > 0 || tombstones_applied > 0 {
+ info!(
+ total,
+ tombstones_applied, "version map rebuilt from segments"
+ );
}
Ok(())
}
@@ -2326,6 +2531,32 @@ impl IndexStore {
}
WalEntry::Delete { doc_id } => {
let seq_no = replay_entry.seq_no;
+ // RC4 W2 #14 — seq-aware replay:
+ //
+ // (a) STALE delete (the map — rebuilt from segments,
+ // including ZTB2 tombstones — already holds a
+ // strictly newer state for the doc): skip.
+ // `VersionMap::delete` is seq-blind; pre-fix a
+ // retained old Delete entry could tombstone a
+ // NEWER segment-resident version whose own WAL
+ // entry had already been pruned.
+ // (b) Tombstone already SEGMENT-resident at >= seq
+ // (rebuilt from a ZTB2 section): the deletion is
+ // durable without this WAL entry — don't re-apply,
+ // don't re-pin, don't re-materialise a memtable
+ // tombstone (which would re-flush it forever).
+ let cur = self.version_map.get(&doc_id);
+ let stale = cur.as_ref().is_some_and(|e| e.seq_no > seq_no);
+ let already_durable = cur.as_ref().is_some_and(|e| {
+ e.deleted && e.seq_no >= seq_no && &*e.segment_id != IN_MEMORY_SEGMENT_ID
+ });
+ if stale || already_durable {
+ let _ = self
+ .seq_counter
+ .fetch_max(replay_entry.seq_no + 1, Ordering::AcqRel);
+ count += 1;
+ continue;
+ }
let applied = self
.version_map
.delete(&doc_id, seq_no, IN_MEMORY_SEGMENT_ID)
@@ -2343,7 +2574,8 @@ impl IndexStore {
// Superseded pins (doc re-indexed later in the replay
// stream) are cleared by the next
// `sweep_pending_deletes` once the newer version is
- // segment-resident.
+ // segment-resident (or once the tombstone itself is
+ // persisted by `persist_pending_tombstones`).
if applied {
self.pending_deletes
.lock()
@@ -2596,11 +2828,7 @@ impl IndexStore {
|| std::env::var("XERJ_STRICT_SYNC")
.map(|v| !v.is_empty() && v != "0")
.unwrap_or(false);
- let sync_result = if strict {
- wal.sync()
- } else {
- wal.soft_flush()
- };
+ let sync_result = if strict { wal.sync() } else { wal.soft_flush() };
wal.set_sync_mode(saved_mode);
sync_result?;
}
@@ -3007,11 +3235,207 @@ mod tests {
}
}
+ /// RC4 W2 #14 regression — a plain, never-re-indexed DELETE must not
+ /// pin its WAL shard forever. Pre-fix: the delete's tombstone lived
+ /// only in RAM + WAL, `sweep_pending_deletes` never unpinned it, and
+ /// maintenance skipped the shard for the life of the process (live
+ /// repro: single-shard WAL grew 866 KB → 3.06 MB across 6 flushed
+ /// rounds, control index pruned to 16 B). Post-fix the flush writes
+ /// a seq-aware ZTB2 tombstone, the version map repoints it
+ /// segment-resident, sweep unpins, prune reclaims — and the delete
+ /// STAYS durable across BOTH the first and second restart (the
+ /// second restart is the resurrection trap: no WAL entry left).
+ #[test]
+ fn plain_delete_unpins_wal_and_stays_deleted_across_restarts() {
+ let dir = tempfile::tempdir().unwrap();
+ {
+ let store = open_test_store(dir.path());
+ for i in 0..50 {
+ store
+ .index(format!("d{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ store.flush().unwrap().expect("segment");
+
+ // The plain delete — tombstone only, never re-indexed.
+ store.delete("d7").unwrap().expect("d7 existed");
+
+ // Flush drains the delete into a ZTB2 tombstone section;
+ // maintenance must then be able to unpin + reclaim the WAL.
+ for i in 50..60 {
+ store
+ .index(format!("d{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ store.flush().unwrap().expect("segment 2");
+ store.force_wal_maintenance().unwrap();
+
+ let files = walk_wal_files(&store.wal_dir());
+ let bytes: u64 = files.iter().sum();
+ assert!(
+ bytes <= files.len() as u64 * 16,
+ "delete-bearing WAL must be reclaimed once the tombstone is \
+ segment-resident (pre-fix: pinned forever): {} files, {bytes} bytes",
+ files.len()
+ );
+ drop(store);
+ }
+
+ // Restart 1: the delete must hold WITHOUT its WAL entry.
+ {
+ let store2 = open_test_store(dir.path());
+ let e = store2.version_map.get("d7");
+ assert!(
+ e.map(|e| e.deleted).unwrap_or(true),
+ "d7 resurrected after restart 1 (tombstone not rebuilt from segment)"
+ );
+ assert!(
+ store2
+ .version_map
+ .get("d8")
+ .map(|e| !e.deleted)
+ .unwrap_or(false),
+ "sibling doc d8 must stay live"
+ );
+ drop(store2);
+ }
+ // Restart 2 (the resurrection trap when durability leaned on a
+ // replayed-then-repinned WAL entry).
+ let store3 = open_test_store(dir.path());
+ assert!(
+ store3
+ .version_map
+ .get("d7")
+ .map(|e| e.deleted)
+ .unwrap_or(true),
+ "d7 resurrected after restart 2"
+ );
+ for i in [0usize, 8, 49, 59] {
+ assert!(
+ store3
+ .version_map
+ .get(&format!("d{i}"))
+ .map(|e| !e.deleted)
+ .unwrap_or(false),
+ "live doc d{i} lost"
+ );
+ }
+ }
+
+ /// RC4 W2 #14 — engine-path shape: the delete's tombstone never flows
+ /// through a data flush (the engine drains its own memtable), so
+ /// maintenance itself must persist it (tombstone-only segment) and
+ /// then reclaim the WAL. Survives restart with the WAL gone.
+ #[test]
+ fn maintenance_persists_pending_tombstones_without_data_flush() {
+ let dir = tempfile::tempdir().unwrap();
+ {
+ let store = open_test_store(dir.path());
+ for i in 0..20 {
+ store
+ .index(format!("e{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ store.flush().unwrap().expect("segment");
+ store.force_wal_maintenance().unwrap();
+
+ // Delete with NO subsequent data flush — only maintenance runs
+ // (the engine flush path never hands deletes to finalize).
+ store.delete("e3").unwrap().expect("e3 existed");
+ store.force_wal_maintenance().unwrap();
+
+ let files = walk_wal_files(&store.wal_dir());
+ let bytes: u64 = files.iter().sum();
+ assert!(
+ bytes <= files.len() as u64 * 16,
+ "maintenance must persist the pending tombstone and reclaim \
+ the WAL: {} files, {bytes} bytes",
+ files.len()
+ );
+ // A tombstone-only segment must be registered.
+ let snap = store.snapshot.load();
+ assert!(
+ snap.segments
+ .iter()
+ .any(|m| m.doc_count == 0 && m.has_tombstones),
+ "tombstone-only segment missing from snapshot"
+ );
+ drop(store);
+ }
+
+ let store2 = open_test_store(dir.path());
+ assert!(
+ store2
+ .version_map
+ .get("e3")
+ .map(|e| e.deleted)
+ .unwrap_or(true),
+ "e3 resurrected after restart (tombstone-only segment not applied)"
+ );
+ assert!(
+ store2
+ .version_map
+ .get("e4")
+ .map(|e| !e.deleted)
+ .unwrap_or(false),
+ "sibling doc e4 must stay live"
+ );
+ }
+
+ /// RC4 W2 #14 — crash-window coverage: a tombstone-only segment left
+ /// ORPHANED (crash between the WAL prune and `save_snapshot`) must be
+ /// resurrected on reopen, not GC'd — with the WAL entry pruned it is
+ /// the only remaining record of the delete.
+ #[test]
+ fn orphan_tombstone_only_segment_is_resurrected() {
+ let dir = tempfile::tempdir().unwrap();
+ let snapshot_backup;
+ {
+ let store = open_test_store(dir.path());
+ for i in 0..10 {
+ store
+ .index(format!("o{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ store.flush().unwrap().expect("segment");
+ store.force_wal_maintenance().unwrap();
+ // Snapshot state BEFORE the delete becomes segment-durable.
+ snapshot_backup = std::fs::read(dir.path().join("snapshot.json")).unwrap();
+
+ store.delete("o5").unwrap().expect("o5 existed");
+ store.force_wal_maintenance().unwrap(); // persists tombstone + prunes WAL
+ drop(store);
+ }
+ // Simulate the crash window: the persisted snapshot never saw the
+ // tombstone-only segment (WAL already pruned).
+ std::fs::write(dir.path().join("snapshot.json"), &snapshot_backup).unwrap();
+
+ let store2 = open_test_store(dir.path());
+ assert!(
+ store2
+ .version_map
+ .get("o5")
+ .map(|e| e.deleted)
+ .unwrap_or(true),
+ "o5 resurrected: orphan tombstone-only segment was not recovered"
+ );
+ assert!(
+ store2
+ .version_map
+ .get("o4")
+ .map(|e| !e.deleted)
+ .unwrap_or(false),
+ "sibling doc o4 must stay live"
+ );
+ }
+
fn walk_wal_files(root: &Path) -> Vec<u64> {
let mut sizes = Vec::new();
let mut dirs = vec![root.to_path_buf()];
while let Some(d) = dirs.pop() {
- let Ok(rd) = std::fs::read_dir(&d) else { continue };
+ let Ok(rd) = std::fs::read_dir(&d) else {
+ continue;
+ };
for e in rd.flatten() {
let p = e.path();
if p.is_dir() {
diff --git a/engine/crates/xerj-storage/src/segment.rs b/engine/crates/xerj-storage/src/segment.rs
index bc40b1f76..6dcc2182c 100644
--- a/engine/crates/xerj-storage/src/segment.rs
+++ b/engine/crates/xerj-storage/src/segment.rs
@@ -127,6 +127,75 @@ impl SectionType {
}
}
+// ── Seq-aware tombstone section codec (RC4 W2 #14) ───────────────────────────
+//
+// `SectionType::Tombstones` payload, version 2:
+//
+// "ZTB2" 4 bytes magic
+// u32 pair count
+// lz4_flex::compress_prepend_size(body) where body repeats:
+// u64 delete seq_no (LE)
+// u16 id_len (LE)
+// id_len bytes UTF-8 doc_id
+//
+// V2 makes acked deletes SEGMENT-DURABLE: reopen applies these pairs
+// max-seq-wins against the doc entries rebuilt from `.ids`/stored, so a
+// delete survives restart without its WAL entry — which is what finally
+// lets WAL maintenance unpin delete-bearing shards (the pre-fix pinning
+// retained one WAL generation per shard FOREVER after a single plain
+// delete). The legacy payload (a bare JSON array of doc_id strings,
+// written since the 2026-07 delete-durability fix but never read) has no
+// seq information and is IGNORED by `decode_tombstones_v2` — those
+// deletes remain protected by WAL pinning exactly as before.
+
+/// Encode `(delete_seq_no, doc_id)` pairs as a ZTB2 tombstone section.
+pub fn encode_tombstones_v2(pairs: &[(u64, &str)]) -> Vec<u8> {
+ let mut body: Vec<u8> =
+ Vec::with_capacity(pairs.iter().map(|(_, id)| 8 + 2 + id.len()).sum::<usize>());
+ for (seq, id) in pairs {
+ body.extend_from_slice(&seq.to_le_bytes());
+ body.extend_from_slice(&(id.len() as u16).to_le_bytes());
+ body.extend_from_slice(id.as_bytes());
+ }
+ let compressed = lz4_flex::compress_prepend_size(&body);
+ let mut out = Vec::with_capacity(8 + compressed.len());
+ out.extend_from_slice(b"ZTB2");
+ out.extend_from_slice(&(pairs.len() as u32).to_le_bytes());
+ out.extend_from_slice(&compressed);
+ out
+}
+
+/// Decode a ZTB2 tombstone section. Returns `None` for the legacy
+/// id-only JSON payload (no seq info — cannot be applied safely) or any
+/// malformed input.
+pub fn decode_tombstones_v2(bytes: &[u8]) -> Option<Vec<(u64, String)>> {
+ if bytes.len() < 8 || &bytes[..4] != b"ZTB2" {
+ return None;
+ }
+ let count = u32::from_le_bytes(bytes[4..8].try_into().ok()?) as usize;
+ let body = lz4_flex::decompress_size_prepended(&bytes[8..]).ok()?;
+ let mut out = Vec::with_capacity(count);
+ let mut pos = 0usize;
+ for _ in 0..count {
+ if pos + 10 > body.len() {
+ return None;
+ }
+ let seq = u64::from_le_bytes(body[pos..pos + 8].try_into().ok()?);
+ pos += 8;
+ let id_len = u16::from_le_bytes(body[pos..pos + 2].try_into().ok()?) as usize;
+ pos += 2;
+ if pos + id_len > body.len() {
+ return None;
+ }
+ let id = std::str::from_utf8(&body[pos..pos + id_len])
+ .ok()?
+ .to_string();
+ pos += id_len;
+ out.push((seq, id));
+ }
+ Some(out)
+}
+
// ── SegmentHeader ─────────────────────────────────────────────────────────────
#[derive(Debug, Clone)]
diff --git a/engine/crates/xerj-storage/src/version_map.rs b/engine/crates/xerj-storage/src/version_map.rs
index 709dddb38..e07134f43 100644
--- a/engine/crates/xerj-storage/src/version_map.rs
+++ b/engine/crates/xerj-storage/src/version_map.rs
@@ -184,6 +184,14 @@ impl VersionMap {
/// resurrect the stale version.
///
/// Atomic per key via the dashmap entry API.
+ ///
+ /// Live/ghost accounting (RC4 W2 #14): applies the same deltas as
+ /// [`set`](Self::set) whenever it actually replaces/creates an entry.
+ /// For the historical merge-repoint caller this is a no-op delta
+ /// (same seq, live→live), so behavior is unchanged there; the
+ /// tombstone-persistence paths (flush repoint, segment-tombstone
+ /// rebuild apply) rely on the accounting being correct for
+ /// live→deleted transitions.
pub fn set_if_latest(
&self,
doc_id: impl Into<String>,
@@ -196,11 +204,19 @@ impl VersionMap {
match self.inner.entry(doc_id) {
Entry::Occupied(mut occ) => {
if seq_no >= occ.get().seq_no {
- occ.insert(VersionEntry {
+ let old = occ.insert(VersionEntry {
seq_no,
segment_id: segment_id.into(),
deleted,
});
+ // Mirror `set`: a same-seq repoint is NOT an overwrite
+ // (see the flush-repoint note there); a tombstone that
+ // REPLACES a live entry is a ghost event.
+ let is_overwrite = old.seq_no != seq_no;
+ if is_overwrite || (deleted && !old.deleted) {
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
+ self.live_delta(!old.deleted, !deleted);
}
}
Entry::Vacant(vac) => {
@@ -209,6 +225,10 @@ impl VersionMap {
segment_id: segment_id.into(),
deleted,
});
+ if deleted {
+ self.ghost_events.fetch_add(1, Ordering::Relaxed);
+ }
+ self.live_delta(false, !deleted);
}
}
}