AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(durability): acked-write loss — verified WAL prune + power-loss-ordered publish chain (RC4 W1 #8+#10)
xerj-org297be608019e
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 7163bccf7..e4b80c3d9 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -5121,6 +5121,7 @@ dependencies = ["prometheus","serde","serde_json",+ "tempfile","thiserror 2.0.18","toml","tracing",diff --git a/engine/crates/xerj-common/Cargo.toml b/engine/crates/xerj-common/Cargo.tomlindex 2b2a70b6a..4ee481307 100644--- a/engine/crates/xerj-common/Cargo.toml+++ b/engine/crates/xerj-common/Cargo.toml@@ -23,3 +23,6 @@ tracing.workspace = truebytes.workspace = truebyteorder.workspace = trueprometheus.workspace = true++[dev-dependencies]+tempfile.workspace = truediff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rsnew file mode 100644index 000000000..7576db517--- /dev/null+++ b/engine/crates/xerj-common/src/fsio.rs@@ -0,0 +1,95 @@+//! Power-loss-ordered file I/O primitives (RC4 Wave-1 blocker #10).+//!+//! The segment publish chain (`.seg` → `.ids`/`.dv`/FTS side-cars →+//! `snapshot.json` → WAL prune) is only crash-safe if every link is+//! durable **before** the WAL entries it supersedes are destroyed.+//! `fs::write` + `rename` alone leaves both the file bytes and the+//! directory entry in the volatile page cache: a power loss after the+//! WAL was pruned can then GC a fully-flushed segment as an orphan —+//! acked-data loss.+//!+//! Every side-car / manifest write on the publish chain must go through+//! [`write_file_durable`], and every `rename` that publishes a file must+//! be followed by [`fsync_dir`] on the parent (rename durability is a+//! property of the *directory*, not the file).++use std::io::Write as _;+use std::path::Path;++/// fsync a directory so previously-renamed/created entries in it survive+/// power loss. No-op errors are surfaced to the caller; on filesystems+/// where directories cannot be fsynced (rare), callers may choose to+/// ignore the error.+pub fn fsync_dir(dir: &Path) -> std::io::Result<()> {+ let d = std::fs::File::open(dir)?;+ d.sync_all()+}++/// Write `bytes` to `path` atomically **and durably**:+///+/// 1. write to a same-directory temp file,+/// 2. `fsync` the temp file (data + metadata),+/// 3. `rename` over the target,+/// 4. `fsync` the parent directory (makes the rename itself durable).+///+/// A crash or power loss at any point leaves either the old file or the+/// complete new file — never a torn one, and never a "file that+/// evaporates on power loss because only the page cache had it".+///+/// The temp name embeds a uuid + thread id so concurrent writers to the+/// same target never clobber each other's temp file (see the+/// `save_snapshot` race note in `xerj-storage`).+pub fn write_file_durable(path: &Path, bytes: &[u8]) -> std::io::Result<()> {+ let nonce = format!(+ "{:x}-{:?}",+ std::process::id(),+ std::thread::current().id()+ );+ let file_name = path+ .file_name()+ .map(|n| n.to_string_lossy().into_owned())+ .unwrap_or_else(|| "file".to_string());+ let tmp = path.with_file_name(format!("{file_name}.tmp.{nonce}"));+ {+ let mut f = std::fs::File::create(&tmp)?;+ f.write_all(bytes)?;+ f.sync_all()?;+ }+ if let Err(e) = std::fs::rename(&tmp, path) {+ let _ = std::fs::remove_file(&tmp);+ return Err(e);+ }+ if let Some(parent) = path.parent() {+ fsync_dir(parent)?;+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn write_file_durable_round_trip() {+ let dir = tempfile::tempdir().unwrap();+ let p = dir.path().join("x.bin");+ write_file_durable(&p, b"hello").unwrap();+ assert_eq!(std::fs::read(&p).unwrap(), b"hello");+ // Overwrite is atomic.+ write_file_durable(&p, b"world").unwrap();+ assert_eq!(std::fs::read(&p).unwrap(), b"world");+ // No stray temp files left behind.+ let leftovers: Vec<_> = std::fs::read_dir(dir.path())+ .unwrap()+ .filter_map(|e| e.ok())+ .filter(|e| e.file_name().to_string_lossy().contains(".tmp."))+ .collect();+ assert!(leftovers.is_empty(), "temp files left: {leftovers:?}");+ }++ #[test]+ fn fsync_dir_works() {+ let dir = tempfile::tempdir().unwrap();+ fsync_dir(dir.path()).unwrap();+ }+}diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rsindex 7964a1eea..80aebf936 100644--- a/engine/crates/xerj-common/src/lib.rs+++ b/engine/crates/xerj-common/src/lib.rs@@ -21,6 +21,7 @@pub mod config;pub mod error;+pub mod fsio;pub mod metrics;pub mod schema;pub mod types;diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex e6433cfc7..ad4ce5e4c 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -8838,17 +8838,23 @@ impl Index {}}}- // Force a global WAL checkpoint + rotate + prune at the user-- // visible flush boundary. `finalize_flush_with_publisher` now- // time-gates this work (1 s window) on the hot per-shard path;- // the final flush must bypass the gate so the CLI session's- // last segment is definitely checkpointed when the CLI exits.- {- let seq = self.store.current_seq_no();- if seq > 0 {- let _ = self.store.force_wal_maintenance(seq.saturating_sub(1));- }- }+ // Force a global WAL maintenance pass (verified checkpoint ++ // rotate + prune) at the user-visible flush boundary.+ // `finalize_flush_with_publisher` time-gates this work (1 s+ // window) on the hot per-shard path; the final flush must bypass+ // the gate so the CLI session's last segment is definitely+ // checkpointed when the CLI exits.+ //+ // RC4 W1 #8 — this used to pass `current_seq_no() - 1` as the+ // checkpoint watermark: the LIVE global counter, covering every+ // acked-but-unflushed doc in existence (docs bulked into other+ // shards, docs that landed between a shard's drain and this+ // call). Maintenance then rotated + pruned their WAL entries+ // while the docs still lived only in memtables — kill -9 lost+ // all of them (live repro: 50/50 acked docs gone). The durable+ // watermark is now derived inside the store from the persisted+ // snapshot, and prune verifies every entry before destroying it.+ let _ = self.store.force_wal_maintenance();// Flushing moves data from memtable → segments, which changes// what the shortcut count paths compute. Bump the dataset// version so the response cache invalidates. P3.2: the new@@ -9404,6 +9410,12 @@ fn pad_with_none<T>(v: &mut Vec<Option<T>>, target_len: usize) {}/// Encode columns to bytes and write them to `{segment_id}.dv`.+///+/// RC4 W1 #10 — durable write (tmp + fsync + rename + dir fsync): the+/// `.dv` side-car is part of the segment publish chain whose WAL entries+/// are pruned ~1 s after the flush; a plain `fs::write` left it in the+/// volatile page cache, so a power loss after the prune produced a+/// segment whose doc-values had evaporated.fn write_doc_values_sidecar(segments_dir: &std::path::Path,segment_id: &str,@@ -9411,7 +9423,7 @@ fn write_doc_values_sidecar() -> std::io::Result<()> {let path = segments_dir.join(format!("{segment_id}.dv"));let bytes = xerj_storage::doc_values::encode_columns(columns);- std::fs::write(&path, bytes)+ xerj_common::fsio::write_file_durable(&path, &bytes)}/// Load `{segment_id}.dv`, returning an empty map if the file doesn'tdiff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rsindex 919c3a266..667c615eb 100644--- a/engine/crates/xerj-fts/src/index.rs+++ b/engine/crates/xerj-fts/src/index.rs@@ -768,7 +768,14 @@ impl FtsIndexWriter {out.extend_from_slice(&compressed);out};- fs::write(&post_path, &post_bytes_wrapped)+ // RC4 W1 #10 — every FTS side-car write below goes through the+ // durable tmp+fsync+rename+dir-fsync pattern. These files are+ // part of the segment publish chain: the WAL entries they cover+ // are pruned ~1 s after the flush, so side-cars sitting only in+ // the volatile page cache meant a power loss could leave a+ // registered segment with missing/torn FTS data (silently wrong+ // query results or unreadable fields) with no WAL to recover from.+ xerj_common::fsio::write_file_durable(&post_path, &post_bytes_wrapped).with_context(|| format!("writing postings to {:?}", post_path))?;// 3. Build and write FST (term → meta byte offset).@@ -782,8 +789,12 @@ impl FtsIndexWriter {// inside the meta's flat array — `meta[offset..offset + 24]`// holds df + ttf + postings_offset + length, so one FST hit →// one bounded mmap read.+ // FST streams into a same-directory temp file; fsync + rename ++ // dir-fsync publishes it durably (RC4 W1 #10 — see the postings+ // note above).+ let fst_tmp = segment_dir.join(format!("{}.{}.fst.tmp", segment_id, field_name));let fst_file = BufWriter::new(- File::create(&fst_path).with_context(|| format!("creating FST file {:?}", fst_path))?,+ File::create(&fst_tmp).with_context(|| format!("creating FST file {:?}", fst_tmp))?,);let mut fst_builder = MapBuilder::new(fst_file).with_context(|| "creating FST builder")?;@@ -799,7 +810,18 @@ impl FtsIndexWriter {.with_context(|| format!("inserting term '{}' into FST", term))?;}}- fst_builder.finish().with_context(|| "finishing FST")?;+ let mut fst_out = fst_builder+ .into_inner()+ .with_context(|| "finishing FST")?;+ fst_out.flush().with_context(|| "flushing FST")?;+ fst_out+ .get_ref()+ .sync_all()+ .with_context(|| "fsyncing FST")?;+ drop(fst_out);+ fs::rename(&fst_tmp, &fst_path)+ .with_context(|| format!("publishing FST {:?}", fst_path))?;+ xerj_common::fsio::fsync_dir(segment_dir).with_context(|| "fsyncing segment dir (fst)")?;// 4. Write meta in the ZFM4 format — Zstd-19 envelope around// the per-term records section; ZFM3-compatible header so@@ -812,7 +834,7 @@ impl FtsIndexWriter {&sorted_terms,&term_postings,)?;- fs::write(&meta_path, &meta_bytes)+ xerj_common::fsio::write_file_durable(&meta_path, &meta_bytes).with_context(|| format!("writing meta to {:?}", meta_path))?;// 5. Write norms file — V4 M4.7 compact format.@@ -852,15 +874,14 @@ impl FtsIndexWriter {(0, &dense[..])};- let norms_file = File::create(&norms_path)- .with_context(|| format!("creating norms file {:?}", norms_path))?;- let mut norms_writer = BufWriter::new(norms_file);- norms_writer.write_all(NORMS_MAGIC)?;- norms_writer.write_u8(encoding)?;- norms_writer.write_u32::<LittleEndian>(dense_len as u32)?;- norms_writer.write_u32::<LittleEndian>(payload.len() as u32)?;- norms_writer.write_all(payload)?;- norms_writer.flush()?;+ let mut norms_bytes: Vec<u8> = Vec::with_capacity(4 + 1 + 4 + 4 + payload.len());+ norms_bytes.extend_from_slice(NORMS_MAGIC);+ norms_bytes.push(encoding);+ norms_bytes.extend_from_slice(&(dense_len as u32).to_le_bytes());+ norms_bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes());+ norms_bytes.extend_from_slice(payload);+ xerj_common::fsio::write_file_durable(&norms_path, &norms_bytes)+ .with_context(|| format!("writing norms to {:?}", norms_path))?;Ok(())}diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 08b0200d1..60b3d8e8a 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -764,7 +764,14 @@ impl IndexStore {.data_dir.join("segments").join(format!("{segment_id}.ids"));- std::fs::write(&ids_path, &buf)+ // RC4 W1 #10 — durable write (tmp + fsync + rename + dir fsync).+ // The `.ids` side-car is the recovery marker+ // `recover_orphaned_segments` relies on when a crash lands between+ // the snapshot publish and the debounced `save_snapshot`; the WAL+ // entries it supersedes are pruned within ~1 s of the flush. A+ // plain `fs::write` left it in the page cache — power loss after+ // the prune GC'd the fully-flushed segment as an orphan.+ xerj_common::fsio::write_file_durable(&ids_path, &buf)}/// Unlink every on-disk file belonging to the given segment ids — the@@ -1490,39 +1497,24 @@ impl IndexStore {// mode. Clean shutdown / explicit `_flush` persists via// `force_wal_maintenance()`.self.save_snapshot()?;- // Delete-durability: unpin deletes subsumed by flushed newer- // versions, then skip maintenance on any shard whose WAL is- // still the only durable record of an acked delete.- self.sweep_pending_deletes();- for (ws_idx, ws) in self.wal_shards.iter().enumerate() {- let mut wal = ws.lock().unwrap();- if self.wal_shard_pinned_by_pending_delete(ws_idx) {- debug!(- shard = ws_idx,- "WAL maintenance skipped: shard pinned by unpersisted delete"- );- continue;- }- wal.checkpoint(max_seq)?;- // Disk-space fix (2026-07): force-rotate + prune instead of- // `rotate_if_large(64 MB)`. The checkpoint we just wrote- // records the shard's FULL current file length (we hold the- // shard mutex, so `offset == current_offset`), i.e. the- // whole active generation is covered — rotating now lets- // `prune()` reclaim it. The old 64 MB threshold never- // fired on realistic workloads: with 16 WAL shards a 1 M-doc- // ingest leaves 10-35 MB per shard file, so ~490 MB of WAL- // that was already durable in segments sat on disk forever- // (the explicit `_flush` path has done force-rotate via- // `force_wal_maintenance` since the 2026-04-25 head-to-head;- // this mirrors it on the background flush path). Rotation- // churn stays bounded by the surrounding 1 s- // `WAL_MAINTENANCE_INTERVAL_MS` gate — at most one- // rotation per shard per second, and `force_rotate` no-ops- // on generations with no data past the header.- wal.force_rotate()?;- wal.prune()?;- }+ // RC4 W1 #8 — verified maintenance (see+ // `wal_maintain_all_verified`). The pre-fix loop here+ // checkpointed EVERY shard with THIS segment's `max_seq` and+ // the shard's full file offset, then force-rotated and pruned+ // — destroying acked-but-unflushed entries on sibling shards+ // (their memtable docs can carry seqs below a fresh segment's+ // max) and on THIS shard (docs appended between the drain and+ // this maintenance tick). Live-verified as the 50/50 kill-9+ // acked-loss repro. The durable watermark passed for the+ // compat checkpoint is the max seq registered in the snapshot+ // we just persisted — never a live counter.+ //+ // Disk-space behaviour is preserved: fully-durable generations+ // still rotate + prune on the same 1 s cadence; generations+ // holding unproven entries are retained (that retention IS the+ // fix) and reclaimed on a later tick once their docs flush.+ let durable_max = self.snapshot.load().max_seq_no;+ self.wal_maintain_all_verified(durable_max)?;}info!(segment_id, doc_count, min_seq, max_seq, "segment flushed");@@ -1613,10 +1605,7 @@ impl IndexStore {/// Delete-durability: true if WAL shard `shard_idx` still holds an/// unpersisted acked delete and therefore MUST NOT be checkpointed,- /// rotated, or pruned (`WalWriter::prune` deletes any rotated- /// generation that has a checkpoint file — irrespective of the- /// checkpoint's `max_seq_no` — so a checkpoint alone already dooms- /// the Delete entries). Callers hold the shard's WAL mutex when+ /// rotated, or pruned. Callers hold the shard's WAL mutex when/// they consult this, which combined with the insert-before-append/// ordering in [`IndexStore::delete`] makes the check race-free.fn wal_shard_pinned_by_pending_delete(&self, shard_idx: usize) -> bool {@@ -1624,61 +1613,127 @@ impl IndexStore {pending.values().any(|&(_, ws)| ws == shard_idx)}- /// Write a checkpoint to ALL WAL shards with the given max_seq_no.- /// Used by `Index::flush` to write a global checkpoint after- /// multi-shard parallel flush.- pub fn wal_checkpoint_all(&self, max_seq: SeqNo) -> Result<()> {+ /// RC4 W1 #8 — the per-entry durability proof used by WAL maintenance.+ ///+ /// Returns true iff destroying this WAL entry cannot lose data:+ ///+ /// - `Index`: the doc is tombstoned at an equal-or-newer seq (the+ /// tombstoning delete is itself WAL-pinned until subsumed, so replay+ /// reconstructs the deletion); OR a strictly newer version of the doc+ /// exists (whose own WAL entry is retained until IT is durable — the+ /// proof chains); OR this exact version has been flushed into a real+ /// segment (version map repointed off `__memtable__`, which happens+ /// only after the segment + its side-cars are durably on disk — see+ /// the blocker-#10 fsync barrier in `finalize_flush_with_publisher`).+ /// - `Delete`: subsumed — the doc was re-indexed strictly newer AND+ /// that version is segment-resident (mirrors+ /// `sweep_pending_deletes`). A load-bearing tombstone is never+ /// prunable.+ /// - `UpdateMapping`: always — `schema.json` is written atomically at+ /// update time and replaying the entry is a no-op.+ ///+ /// `None` from the version map is conservatively NOT durable: it can+ /// mean "version map not yet updated for a just-appended doc" (the+ /// batch paths append to the WAL before the version-map set).+ fn wal_entry_durable(&self, entry: &WalEntry, seq: SeqNo) -> bool {+ match entry {+ WalEntry::Index { doc_id, .. } => match self.version_map.get(doc_id) {+ Some(e) if e.deleted => e.seq_no >= seq,+ Some(e) => {+ e.seq_no > seq+ || (e.seq_no == seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID)+ }+ None => false,+ },+ WalEntry::Delete { doc_id } => match self.version_map.get(doc_id) {+ Some(e) if !e.deleted => {+ e.seq_no > seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID+ }+ _ => false,+ },+ WalEntry::UpdateMapping { .. } => true,+ }+ }++ /// RC4 W1 #8 — verified WAL maintenance across all shards.+ ///+ /// Replaces the pre-fix `checkpoint(global_max_seq) + force_rotate ++ /// prune` loop, which destroyed acked-but-unflushed entries two ways:+ ///+ /// 1. The checkpoint was written with a GLOBAL max_seq+ /// (`current_seq_no()-1` from `Index::flush`, or a sibling shard's+ /// segment max) and the shard's FULL current offset — covering+ /// entries whose docs still lived only in a memtable. Replay then+ /// skipped them (loss channel closed by making replay ignore+ /// checkpoints), and+ /// 2. `prune()` deleted any rotated generation that had a checkpoint+ /// file, destroying those same entries outright (loss channel+ /// closed by per-entry verified pruning).+ ///+ /// New per-shard flow (under the shard's WAL mutex):+ /// - skip entirely if the shard is pinned by an unpersisted delete;+ /// - decode the ACTIVE generation and check every entry against+ /// [`wal_entry_durable`](Self::wal_entry_durable);+ /// - if fully durable: write a checkpoint (safe values — kept for+ /// data-dir compatibility with older binaries) and force-rotate so+ /// the generation becomes prunable;+ /// - if it holds any unproven entry: force-rotate WITHOUT a+ /// checkpoint (freezes the generation; per-entry verification+ /// reclaims it on a later tick once everything in it flushed);+ /// - `prune_verified` every rotated generation.+ ///+ /// The caller must persist the snapshot (`save_snapshot`) BEFORE this+ /// runs so every segment the proofs point at is registered on disk.+ fn wal_maintain_all_verified(&self, durable_max_seq: SeqNo) -> Result<()> {self.sweep_pending_deletes();for (ws_idx, ws) in self.wal_shards.iter().enumerate() {let mut wal = ws.lock().unwrap();- // Delete-durability: a checkpoint alone already marks the- // shard's current generation prunable, so pinned shards must- // skip it (see `wal_shard_pinned_by_pending_delete`).if self.wal_shard_pinned_by_pending_delete(ws_idx) {+ debug!(+ shard = ws_idx,+ "WAL maintenance skipped: shard pinned by unpersisted delete"+ );continue;}- wal.checkpoint(max_seq)?;+ // Drain the userspace buffer so the on-disk active generation+ // is complete before we decode it.+ wal.soft_flush()?;+ let active_gen = wal.active_generation();+ let (entries, clean) = wal.read_generation_entries(active_gen);+ let all_durable =+ clean && entries.iter().all(|e| self.wal_entry_durable(&e.entry, e.seq_no));+ if all_durable {+ wal.checkpoint(durable_max_seq)?;+ wal.force_rotate()?;+ } else if !entries.is_empty() {+ wal.force_rotate()?;+ }+ wal.prune_verified(&|entry, seq| self.wal_entry_durable(entry, seq))?;}Ok(())}- /// Unconditionally run WAL maintenance (checkpoint + force-rotate +- /// prune) across all shards. Bypasses the- /// `WAL_MAINTENANCE_INTERVAL_MS` gate that+ /// Unconditionally run verified WAL maintenance across all shards.+ /// Bypasses the `WAL_MAINTENANCE_INTERVAL_MS` gate that/// `finalize_flush_with_publisher` uses on the hot flush path./// Called by `Index::flush()` (the final drain / user-triggered/// `_flush`) so disk cleanup happens immediately — matches ES's- /// `_flush`-time translog rollover semantics where every user- /// flush rotates and prunes. The periodic background tick- /// continues to use the 64 MB threshold (amortises rotation- /// churn under sustained ingest), but explicit user flushes get- /// the unconditional rotation so disk gets reclaimed promptly.- pub fn force_wal_maintenance(&self, max_seq: SeqNo) -> Result<()> {+ /// `_flush`-time translog rollover semantics.+ ///+ /// RC4 W1 #8 — signature change: the old `max_seq: SeqNo` parameter+ /// is gone. `Index::flush` passed `current_seq_no() - 1`, which+ /// covered every acked-but-unflushed doc in existence and was the+ /// direct trigger of the 50/50 kill-9 loss repro. The durable+ /// watermark is now computed internally from the persisted snapshot+ /// (max seq actually resident in flushed segments).+ pub fn force_wal_maintenance(&self) -> Result<()> {// P2.3 — persist the (possibly debounced) snapshot before pruning// so an explicit `_flush` / clean shutdown always leaves the// on-disk snapshot covering every segment whose WAL is about to// be dropped. Mirrors the coupling in the gated flush path.self.save_snapshot()?;- // Delete-durability: same shard-pinning rule as the gated- // background path — an explicit `_flush`/`_refresh`/shutdown- // flush must not destroy the WAL Delete entries that are the- // only durable record of acked deletes (pre-fix this very call,- // reached from `Index::flush` — including the SIGTERM final- // flush — pruned them and deleted docs resurrected on restart).- self.sweep_pending_deletes();- for (ws_idx, ws) in self.wal_shards.iter().enumerate() {- let mut wal = ws.lock().unwrap();- if self.wal_shard_pinned_by_pending_delete(ws_idx) {- debug!(- shard = ws_idx,- "forced WAL maintenance skipped: shard pinned by unpersisted delete"- );- continue;- }- wal.checkpoint(max_seq)?;- wal.force_rotate()?;- wal.prune()?;- }+ let durable_max = self.snapshot.load().max_seq_no;+ self.wal_maintain_all_verified(durable_max)?;let now_ms = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_millis() as u64)@@ -1867,13 +1922,28 @@ impl IndexStore {std::thread::current().id(),);let tmp = path.with_extension(format!("tmp.{nonce}"));- std::fs::write(&tmp, &bytes)?;+ // RC4 W1 #10 — the snapshot is the manifest that makes flushed+ // segments discoverable on restart, and WAL maintenance prunes the+ // covered entries immediately after this returns. Both the file+ // bytes and the rename must therefore be durable BEFORE the prune+ // barrier: write + fsync the tmp, rename, fsync the directory.+ // Pre-fix (`fs::write` + `rename`, no fsync anywhere) a power loss+ // within the writeback window could leave an old/absent+ // snapshot.json next to an already-pruned WAL — flushed segments+ // then got GC'd as orphans on reopen (acked-data loss).+ {+ use std::io::Write as _;+ let mut f = std::fs::File::create(&tmp)?;+ f.write_all(&bytes)?;+ f.sync_all()?;+ }// Atomic rename onto the real path. Last writer wins on the// final snapshot contents, but that's fine: each caller sees// the same `self.snapshot.load()` atomically-swapped payload// (arc_swap), so there is no content-level race — only the// filesystem tmp name was the contention source.std::fs::rename(&tmp, &path)?;+ xerj_common::fsio::fsync_dir(&self.data_dir)?;Ok(())}@@ -2564,6 +2634,137 @@ mod tests {.unwrap()}+ /// RC4 W1 #8 regression — bulk-during-flush + kill -9 must lose ZERO+ /// acked writes.+ ///+ /// Live repro this encodes (stream C evidence, 2026-07-12): 50 000-doc+ /// bulk A, `_flush` dispatched, 50-doc bulk B acked while A's flush+ /// finalize was in flight, kill -9 after the flush returned, restart →+ /// `total=50000`, bulk-B survivors **0/50**. Root cause: flush-time+ /// WAL maintenance checkpointed every shard with a global max_seq+ /// (`current_seq_no()-1` / the fresh segment's max) + full file+ /// offset, then `prune()` deleted any checkpointed generation — B's+ /// WAL entries (memtable-only) were destroyed while B sat in RAM.+ ///+ /// In-process kill simulation: `drop(store)` without a flush is+ /// equivalent to SIGKILL for durability purposes — the memtable (RAM)+ /// is gone, and the WAL bytes that survive are exactly those the+ /// appends already pushed to the kernel (every batched append+ /// soft-flushes, so the userspace buffer is empty at kill time).+ #[test]+ fn bulk_during_flush_kill9_loses_zero_acked_writes() {+ let dir = tempfile::tempdir().unwrap();+ {+ let store = open_test_store(dir.path());++ // Bulk A: acked, then drained for a flush (segment write "in+ // flight" — this is the `_flush` racing point).+ for i in 0..100 {+ store+ .index(format!("a{i}"), serde_json::json!({"v": i}))+ .unwrap();+ }+ let drained = store.take_memtable_for_flush().unwrap();++ // Bulk B: 50 docs acked AFTER the drain — they miss the+ // in-flight segment and live only in WAL + memtable.+ for i in 0..50 {+ store+ .index(format!("b{i}"), serde_json::json!({"v": 100000 + i}))+ .unwrap();+ }++ // Finalize A's flush — runs the gated WAL maintenance+ // (pre-fix: checkpoint covering B + rotate + prune = B's WAL+ // destroyed).+ store+ .finalize_flush_with_publisher(drained, |_| Ok(()))+ .unwrap();+ // The user-visible flush boundary forces maintenance again+ // (pre-fix with `current_seq_no() - 1`).+ store.force_wal_maintenance().unwrap();++ // kill -9.+ drop(store);+ }++ // Restart: every acked write must be recoverable.+ let store2 = open_test_store(dir.path());+ for i in 0..100 {+ let e = store2.version_map.get(&format!("a{i}"));+ assert!(+ e.map(|e| !e.deleted).unwrap_or(false),+ "flushed doc a{i} lost after kill+restart"+ );+ }+ let mut lost = Vec::new();+ for i in 0..50 {+ let alive = store2+ .version_map+ .get(&format!("b{i}"))+ .map(|e| !e.deleted)+ .unwrap_or(false);+ if !alive {+ lost.push(i);+ }+ }+ assert!(+ lost.is_empty(),+ "acked bulk-during-flush docs lost after kill -9: {}/50 (ids {:?})",+ lost.len(),+ lost+ );+ }++ /// RC4 W1 #8 — after everything is flushed and maintenance runs, the+ /// verified prune must still reclaim the WAL (no retention leak from+ /// the new conservatism).+ #[test]+ fn verified_prune_still_reclaims_fully_flushed_wal() {+ let dir = tempfile::tempdir().unwrap();+ let store = open_test_store(dir.path());+ for i in 0..100 {+ store+ .index(format!("d{i}"), serde_json::json!({"v": i}))+ .unwrap();+ }+ store.flush().unwrap().expect("segment");+ store.force_wal_maintenance().unwrap();++ // All docs are segment-resident → every generation must be gone+ // except the fresh empty active one.+ let wal_dir = store.wal_dir();+ let mut wal_bytes = 0u64;+ let mut wal_files = 0usize;+ for entry in walk_wal_files(&wal_dir) {+ wal_files += 1;+ wal_bytes += entry;+ }+ // Each surviving file may only be an empty active generation+ // (16-byte header).+ assert!(+ wal_bytes <= wal_files as u64 * 16,+ "fully-flushed WAL not reclaimed: {wal_files} files, {wal_bytes} bytes"+ );+ }++ fn walk_wal_files(root: &Path) -> Vec<u64> {+ let mut sizes = Vec::new();+ let mut dirs = vec![root.to_path_buf()];+ while let Some(d) = dirs.pop() {+ let Ok(rd) = std::fs::read_dir(&d) else { continue };+ for e in rd.flatten() {+ let p = e.path();+ if p.is_dir() {+ dirs.push(p);+ } else if p.extension().map(|x| x == "wal").unwrap_or(false) {+ sizes.push(e.metadata().map(|m| m.len()).unwrap_or(0));+ }+ }+ }+ sizes+ }+#[test]fn index_and_flush() {let dir = tempfile::tempdir().unwrap();diff --git a/engine/crates/xerj-storage/src/segment.rs b/engine/crates/xerj-storage/src/segment.rsindex d6c94ce9e..bc40b1f76 100644--- a/engine/crates/xerj-storage/src/segment.rs+++ b/engine/crates/xerj-storage/src/segment.rs@@ -419,6 +419,13 @@ impl SegmentWriter {}std::fs::rename(&sidx_tmp, &sidx_path)?;+ // RC4 W1 #10 — make the renames themselves power-loss durable.+ // The file CONTENTS were fsynced above, but a rename only becomes+ // durable when the parent directory is fsynced; without this a+ // power loss could leave a fully-written segment invisible in the+ // directory while the WAL entries it covers were already pruned.+ xerj_common::fsio::fsync_dir(&self.dir)?;+debug!(id = %self.id, size = buf.len(), doc_count, "segment written");Ok(SegmentMeta {diff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rsindex e7e176084..d3d696162 100644--- a/engine/crates/xerj-storage/src/wal.rs+++ b/engine/crates/xerj-storage/src/wal.rs@@ -24,8 +24,8 @@//!//! ## Checkpoint file (`<gen>.wchk`)//!-//! Written after a successful segment flush; used to skip already-flushed-//! entries during replay.+//! Written after a successful segment flush **only when every entry of the+//! generation is provably durable** (see the RC4 W1 #8 note below).//!//! ```text//! generation u64@@ -37,8 +37,33 @@//! ## Generation rotation//!//! When the current WAL file exceeds `wal_max_size_bytes` the writer opens a-//! new file with `generation + 1`. Old files whose entries are fully covered-//! by the checkpoint are deleted by [`WalWriter::prune`].+//! new file with `generation + 1`. Old generations are deleted by+//! [`WalWriter::prune_verified`] once **every entry they hold** is verified+//! durable-or-superseded.+//!+//! ## Durability contract (RC4 Wave-1 blocker #8)+//!+//! A WAL byte may be destroyed (pruned) or ignored (skipped at replay)+//! **only** if the entry it belongs to is provably durable in a flushed+//! segment or superseded by a newer durable/replayable version. Two+//! mechanisms enforce this:+//!+//! 1. **Replay replays everything.** Checkpoints are no longer used to+//! skip entries at replay: both replay consumers (the storage memtable+//! rebuild and the engine FTS memtable rebuild) are idempotent — they+//! check the segment-rebuilt version map and skip entries whose doc is+//! already segment-resident at an equal-or-newer seq. The pre-fix skip+//! rule (`positionally-covered && seq_no <= checkpoint.max_seq_no`)+//! silently discarded acked-but-unflushed entries whenever a checkpoint+//! over-covered — e.g. the flush path checkpointing+//! `current_seq_no()-1` while other shards' memtables still held acked+//! docs with smaller seqs. Live-verified as 50/50 acked-doc loss on+//! kill -9 racing `_flush` (RC4 review, stream C).+//!+//! 2. **Prune verifies per entry.** [`WalWriter::prune_verified`] decodes+//! every entry of a candidate generation and deletes the file only if+//! the caller-supplied verifier proves every single entry durable. A+//! generation holding even one acked-but-unflushed entry is retained.use std::fs::{self, File, OpenOptions};use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write};@@ -567,60 +592,112 @@ impl WalWriter {Ok(())}- /// Remove WAL generations whose entries are fully covered by a checkpoint.+ /// Current active generation number.+ pub fn active_generation(&self) -> u64 {+ self.generation+ }++ /// Read every decodable entry of generation `gen` from disk.+ ///+ /// Returns `(entries, clean)` where `clean == true` means the file was+ /// decoded to the end without a single framing/CRC/JSON error. Callers+ /// verifying prunability MUST treat `clean == false` as "cannot prove+ /// anything about the undecodable tail" and retain the generation.///- /// V4 M4 — extended semantics: any generation file with `gen < self.generation`- /// that has its own checkpoint (`gen.wchk`) is safe to delete because the- /// checkpoint guarantees all its entries are already durable in segments.- /// This lets `finalize_flush_with_publisher → rotate_if_large → prune`- /// actually reclaim space instead of keeping old WAL files indefinitely.- pub fn prune(&self) -> Result<()> {+ /// For the ACTIVE generation the caller must hold the WAL mutex (this+ /// is a `&self` read of an append-only file; the writer soft-flushes+ /// on every append so the on-disk prefix is complete up to+ /// `current_offset`).+ pub fn read_generation_entries(&self, gen: u64) -> (Vec<ReplayEntry>, bool) {+ let path = wal_path(&self.dir, gen);+ if !path.exists() {+ return (Vec::new(), true);+ }+ let results = read_wal_file(path);+ let clean = results.iter().all(|r| r.is_ok());+ let entries = results.into_iter().filter_map(|r| r.ok()).collect();+ (entries, clean)+ }++ /// RC4 W1 #8 — remove WAL generations whose entries are **all verified+ /// durable-or-superseded** by the caller-supplied predicate.+ ///+ /// This replaces the pre-fix `prune()` whose rule was "gen < active &&+ /// gen has a checkpoint file". That rule destroyed acked-but-unflushed+ /// entries: the flush path checkpointed every shard with a *global*+ /// max_seq (`current_seq_no()-1`, or a sibling shard's segment max)+ /// and a full-file offset, so a generation still holding entries whose+ /// docs lived only in the memtable got a checkpoint and was deleted —+ /// kill -9 then lost every one of those acked docs (live-verified+ /// 50/50 loss).+ ///+ /// New rule: decode every entry of every `gen < active_generation` and+ /// delete the file only when+ /// 1. the file decodes cleanly end-to-end (no torn/corrupt frames), and+ /// 2. `verify(entry, seq_no)` returns true for EVERY entry.+ ///+ /// The verifier is supplied by `IndexStore` and proves an entry durable+ /// via the version map (doc segment-resident at `>=` seq, superseded by+ /// a newer retained version, or tombstoned by a retained delete).+ ///+ /// ENOENT on `remove_file` is benign: sharded flushes run in parallel+ /// and every one of them prunes ALL WAL shards; two concurrent+ /// maintenance passes can race to delete the same file — the loser+ /// gets NotFound.+ ///+ /// Returns the number of generations pruned.+ pub fn prune_verified(+ &self,+ verify: &dyn Fn(&WalEntry, SeqNo) -> bool,+ ) -> Result<usize> {let active_gen = self.generation;let entries = fs::read_dir(&self.dir)?;- // Collect (gen, path, is_wal_file, is_checkpoint_file) tuples.let mut wal_files: Vec<(u64, std::path::PathBuf)> = Vec::new();- let mut chkpt_files: std::collections::HashSet<u64> = std::collections::HashSet::new();for entry in entries.flatten() {let name = entry.file_name();let name_str = name.to_string_lossy();if name_str.ends_with(".wal") {if let Some(gen) = parse_wal_generation(&name_str) {- wal_files.push((gen, entry.path()));- }- } else if let Some(stem) = name_str.strip_suffix(".wchk") {- if let Ok(gen) = u64::from_str_radix(stem, 16) {- chkpt_files.insert(gen);+ if gen < active_gen {+ wal_files.push((gen, entry.path()));+ }}}}- // Delete every .wal file whose generation is strictly older than- // the active generation AND has its own checkpoint. Also delete- // the stale checkpoint files once their corresponding wal is gone.- //- // ENOENT on `remove_file` is benign: sharded flushes run in- // parallel and every one of them calls `prune()` across ALL WAL- // shards (see `finalize_flush_with_publisher`). Two concurrent- // flushes can both see the same old-generation .wal file in- // `read_dir` and race to delete it; the loser gets NotFound.- // Pre-fix, that loser's whole flush aborted with "No such file"- // and its shard's data was left in the memtable for the next- // tick — measurable as ~1/5 shard-flushes failing on sustained- // 20 M ingest.+ let mut pruned = 0usize;for (gen, path) in &wal_files {- if *gen < active_gen && chkpt_files.contains(gen) {- match fs::remove_file(path) {- Ok(()) => debug!(gen, "pruned WAL file"),- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {- debug!(gen, "WAL file already pruned by sibling flush — ok");- }- Err(e) => return Err(e.into()),+ let (gen_entries, clean) = self.read_generation_entries(*gen);+ if !clean {+ // Undecodable bytes — cannot prove them durable; keep the+ // generation. (Torn tails from a previous crash park here+ // until the Wave-2 torn-tail truncation lands.)+ debug!(gen, "WAL generation retained: undecodable entries");+ continue;+ }+ let all_durable = gen_entries.iter().all(|e| verify(&e.entry, e.seq_no));+ if !all_durable {+ debug!(+ gen,+ entries = gen_entries.len(),+ "WAL generation retained: holds acked-but-unflushed entries"+ );+ continue;+ }+ match fs::remove_file(path) {+ Ok(()) => {+ pruned += 1;+ debug!(gen, entries = gen_entries.len(), "pruned WAL generation");+ }+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {+ debug!(gen, "WAL file already pruned by sibling flush — ok");}- let ck = checkpoint_path(&self.dir, *gen);- let _ = fs::remove_file(&ck);+ Err(e) => return Err(e.into()),}+ let ck = checkpoint_path(&self.dir, *gen);+ let _ = fs::remove_file(&ck);}- Ok(())+ Ok(pruned)}fn rotate(&mut self) -> Result<()> {@@ -748,44 +825,34 @@ impl WalReader {}}- /// Return an iterator of all entries that are NOT yet covered by a- /// checkpoint, ordered by sequence number.+ /// Return an iterator over **every** entry in every surviving WAL+ /// generation, ordered by sequence number within each file.+ ///+ /// RC4 W1 #8 — replay no longer consults checkpoints to skip entries.+ /// Both replay consumers (`IndexStore::replay_wal` and the engine FTS+ /// memtable rebuild) are idempotent: they skip entries whose doc is+ /// already segment-resident at an equal-or-newer seq via the+ /// segment-rebuilt version map. Skipping at the WAL layer, by+ /// contrast, silently DISCARDED acked-but-unflushed entries whenever a+ /// checkpoint over-covered (stale checkpoints, global max_seq written+ /// onto per-shard files, full-file offsets recorded while sibling+ /// memtables still held acked docs). Surviving generations are exactly+ /// the not-yet-verified-durable set (see `prune_verified`), so the+ /// replay cost of reading them in full is proportional to the data+ /// that genuinely needs recovery.pub fn replay(&self) -> Result<impl Iterator<Item = Result<ReplayEntry>> + '_> {- let (latest_gen, _) = find_latest_generation(&self.dir)?;-- // Determine the starting generation and offset from the checkpoint- let checkpoint = match read_checkpoint(&self.dir, latest_gen) {- Ok(c) => {- // debug: fires per WAL (16 shards × every index) at startup.- debug!(- generation = c.generation,- max_seq_no = c.max_seq_no,- offset = c.offset,- "replaying from checkpoint"- );- Some(c)- }- Err(_) => {- debug!("no checkpoint found, replaying from generation 0");- None- }- };- let start_gen = checkpoint.as_ref().map(|c| c.generation).unwrap_or(0);-- // Collect all generations >= start_genlet mut gens: Vec<u64> = fs::read_dir(&self.dir).into_iter().flatten().flatten().filter_map(|e| parse_wal_generation(&e.file_name().to_string_lossy()))- .filter(|&g| g >= start_gen).collect();gens.sort_unstable();let dir = self.dir.clone();let iter = gens.into_iter().flat_map(move |gen| {let path = wal_path(&dir, gen);- read_wal_file(path, checkpoint)+ read_wal_file(path)});Ok(iter)@@ -854,22 +921,16 @@ pub fn replay_all_sorted(wal_root: &Path) -> Vec<ReplayEntry> {all_entries}-/// Read all entries from a single WAL file, skipping those already covered-/// by `checkpoint`.+/// Read all entries from a single WAL file.///-/// OFFSET-BOUNDED SKIP (2026-07, S3 hardening): an entry is "covered" only-/// when it sits BEFORE the checkpointed byte offset of the checkpointed-/// generation (or in an older generation). A checkpoint by construction-/// covers exactly the bytes `[0, checkpoint.offset)` of its generation —-/// anything appended past that offset was written AFTER the checkpoint and-/// must replay regardless of its seq_no. The old rule (`seq_no <=-/// checkpoint.max_seq_no`, applied file-wide) silently discarded every-/// post-restart acked op whenever the seq counter had regressed across a-/// restart (stale .wchk with max_seq_no=X on the active generation, tail-/// entries seq 1..K <= X) — 100% loss of the acked tail. It also protects-/// against the flush-checkpoint race where a reserved-but-unflushed seq-/// range is covered by a concurrent checkpoint's max_seq.-fn read_wal_file(path: PathBuf, checkpoint: Option<WalCheckpoint>) -> Vec<Result<ReplayEntry>> {+/// RC4 W1 #8: checkpoint-based skipping is GONE from replay (see+/// `WalReader::replay` docs). The 2026-07 S3 "offset-bounded skip"+/// hardening was the previous, weaker containment of the same bug class+/// (stale/over-broad checkpoints discarding acked entries); removing the+/// skip entirely is its logical conclusion — replay-side loss is now+/// structurally impossible, and dedup of already-flushed entries is the+/// job of the idempotent replay consumers.+fn read_wal_file(path: PathBuf) -> Vec<Result<ReplayEntry>> {let file = match File::open(&path) {Ok(f) => f,Err(e) => return vec![Err(e.into())],@@ -957,16 +1018,6 @@ fn read_wal_file(path: PathBuf, checkpoint: Option<WalCheckpoint>) -> Vec<Resultlet this_offset = file_offset;file_offset += entry_total;- // Skip entries already covered by the checkpoint — see the- // offset-bounded rule in the function docs.- if let Some(c) = &checkpoint {- let positionally_covered =- generation < c.generation || (generation == c.generation && this_offset < c.offset);- if positionally_covered && seq_no <= c.max_seq_no {- continue;- }- }-// Check and strip the compression flag from op_code.let is_compressed = (op_code & OP_COMPRESSED_FLAG) != 0;let raw_op_code = op_code & !OP_COMPRESSED_FLAG;@@ -1123,6 +1174,11 @@ fn create_wal_file(path: &Path, generation: u64) -> io::Result<File> {OpenOptions::new().append(true).open(path)}+/// Read a generation's checkpoint file. Replay no longer consumes+/// checkpoints (RC4 W1 #8); they are still WRITTEN — with verified-safe+/// values only — for data-dir compatibility with older binaries whose+/// replay/prune still read them. Kept for tests and offline tooling.+#[allow(dead_code)]fn read_checkpoint(dir: &Path, generation: u64) -> io::Result<WalCheckpoint> {let path = checkpoint_path(dir, generation);let mut f = File::open(&path)?;@@ -1180,7 +1236,13 @@ mod tests {}#[test]- fn checkpoint_skips_old_entries() {+ fn replay_ignores_checkpoints_and_returns_everything() {+ // RC4 W1 #8: checkpoints must never cause replay to discard+ // entries. Pre-fix, a checkpoint with max_seq_no=3 made replay+ // skip seqs 1..3 — if any of those were acked-but-unflushed+ // (over-broad checkpoint), they were silently lost. Replay now+ // returns every surviving entry; dedup of genuinely-flushed docs+ // is the idempotent replay consumers' job (version-map guarded).let dir = tempfile::tempdir().unwrap();let seq_ctr = Arc::new(AtomicU64::new(1));let mut w = WalWriter::open(@@ -1197,15 +1259,66 @@ mod tests {}).unwrap();}- // Checkpoint after seq 3+ // A (deliberately over-broad) checkpoint after seq 3.w.checkpoint(3).unwrap();drop(w);let reader = WalReader::new(dir.path());let entries: Vec<_> = reader.replay().unwrap().collect();- // Only seq 4 and 5 should be returned (seq_no=4,5 since we started at 1)- assert!(entries.iter().all(|e| e.as_ref().unwrap().seq_no > 3));- assert_eq!(entries.len(), 2);+ assert_eq!(entries.len(), 5, "replay must return ALL entries");+ }++ #[test]+ fn prune_verified_retains_generations_with_unflushed_entries() {+ // RC4 W1 #8 core regression at the WAL layer: a rotated+ // generation holding even one entry the verifier cannot prove+ // durable must survive prune; fully-verified generations are+ // deleted.+ let dir = tempfile::tempdir().unwrap();+ let seq_ctr = Arc::new(AtomicU64::new(1));+ let mut w = WalWriter::open(+ dir.path(),+ 64 * 1024 * 1024,+ SyncMode::Batched,+ Arc::clone(&seq_ctr),+ )+ .unwrap();++ // Gen 0: seqs 1..=3. Rotate. Gen 1: seqs 4..=5. Rotate.+ for i in 0..3 {+ w.append(&WalEntry::Delete {+ doc_id: format!("g0-{i}"),+ })+ .unwrap();+ }+ w.force_rotate().unwrap();+ for i in 0..2 {+ w.append(&WalEntry::Delete {+ doc_id: format!("g1-{i}"),+ })+ .unwrap();+ }+ w.force_rotate().unwrap();++ // Verifier: everything with seq <= 3 is durable; seq 4+ is not+ // (i.e. gen 1 holds acked-but-unflushed entries).+ let pruned = w.prune_verified(&|_e, seq| seq <= 3).unwrap();+ assert_eq!(pruned, 1, "exactly gen 0 must be pruned");++ // Gen 1's entries must still replay in full.+ let reader = WalReader::new(dir.path());+ let survivors: Vec<_> = reader+ .replay()+ .unwrap()+ .map(|r| r.unwrap().seq_no)+ .collect();+ assert_eq!(survivors, vec![4, 5], "unverified generation retained");++ // Once the verifier can prove everything, the rest is reclaimed.+ let pruned2 = w.prune_verified(&|_e, _seq| true).unwrap();+ assert_eq!(pruned2, 1);+ let reader = WalReader::new(dir.path());+ assert_eq!(reader.replay().unwrap().count(), 0);}#[test]