Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(durability): acked-write loss — verified WAL prune + power-loss-ordered publish chain (RC4 W1 #8+#10)

xerj-org297be608019e
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 7163bccf7..e4b80c3d9 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -5121,6 +5121,7 @@ dependencies = [
"prometheus",
"serde",
"serde_json",
+ "tempfile",
"thiserror 2.0.18",
"toml",
"tracing",
diff --git a/engine/crates/xerj-common/Cargo.toml b/engine/crates/xerj-common/Cargo.toml
index 2b2a70b6a..4ee481307 100644
--- a/engine/crates/xerj-common/Cargo.toml
+++ b/engine/crates/xerj-common/Cargo.toml
@@ -23,3 +23,6 @@ tracing.workspace = true
bytes.workspace = true
byteorder.workspace = true
prometheus.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
diff --git a/engine/crates/xerj-common/src/fsio.rs b/engine/crates/xerj-common/src/fsio.rs
new file mode 100644
index 000000000..7576db517
--- /dev/null
+++ b/engine/crates/xerj-common/src/fsio.rs
@@ -0,0 +1,95 @@
+//! Power-loss-ordered file I/O primitives (RC4 Wave-1 blocker #10).
+//!
+//! The segment publish chain (`.seg` → `.ids`/`.dv`/FTS side-cars →
+//! `snapshot.json` → WAL prune) is only crash-safe if every link is
+//! durable **before** the WAL entries it supersedes are destroyed.
+//! `fs::write` + `rename` alone leaves both the file bytes and the
+//! directory entry in the volatile page cache: a power loss after the
+//! WAL was pruned can then GC a fully-flushed segment as an orphan —
+//! acked-data loss.
+//!
+//! Every side-car / manifest write on the publish chain must go through
+//! [`write_file_durable`], and every `rename` that publishes a file must
+//! be followed by [`fsync_dir`] on the parent (rename durability is a
+//! property of the *directory*, not the file).
+
+use std::io::Write as _;
+use std::path::Path;
+
+/// fsync a directory so previously-renamed/created entries in it survive
+/// power loss. No-op errors are surfaced to the caller; on filesystems
+/// where directories cannot be fsynced (rare), callers may choose to
+/// ignore the error.
+pub fn fsync_dir(dir: &Path) -> std::io::Result<()> {
+ let d = std::fs::File::open(dir)?;
+ d.sync_all()
+}
+
+/// Write `bytes` to `path` atomically **and durably**:
+///
+/// 1. write to a same-directory temp file,
+/// 2. `fsync` the temp file (data + metadata),
+/// 3. `rename` over the target,
+/// 4. `fsync` the parent directory (makes the rename itself durable).
+///
+/// A crash or power loss at any point leaves either the old file or the
+/// complete new file — never a torn one, and never a "file that
+/// evaporates on power loss because only the page cache had it".
+///
+/// The temp name embeds a uuid + thread id so concurrent writers to the
+/// same target never clobber each other's temp file (see the
+/// `save_snapshot` race note in `xerj-storage`).
+pub fn write_file_durable(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
+ let nonce = format!(
+ "{:x}-{:?}",
+ std::process::id(),
+ std::thread::current().id()
+ );
+ let file_name = path
+ .file_name()
+ .map(|n| n.to_string_lossy().into_owned())
+ .unwrap_or_else(|| "file".to_string());
+ let tmp = path.with_file_name(format!("{file_name}.tmp.{nonce}"));
+ {
+ let mut f = std::fs::File::create(&tmp)?;
+ f.write_all(bytes)?;
+ f.sync_all()?;
+ }
+ if let Err(e) = std::fs::rename(&tmp, path) {
+ let _ = std::fs::remove_file(&tmp);
+ return Err(e);
+ }
+ if let Some(parent) = path.parent() {
+ fsync_dir(parent)?;
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn write_file_durable_round_trip() {
+ let dir = tempfile::tempdir().unwrap();
+ let p = dir.path().join("x.bin");
+ write_file_durable(&p, b"hello").unwrap();
+ assert_eq!(std::fs::read(&p).unwrap(), b"hello");
+ // Overwrite is atomic.
+ write_file_durable(&p, b"world").unwrap();
+ assert_eq!(std::fs::read(&p).unwrap(), b"world");
+ // No stray temp files left behind.
+ let leftovers: Vec<_> = std::fs::read_dir(dir.path())
+ .unwrap()
+ .filter_map(|e| e.ok())
+ .filter(|e| e.file_name().to_string_lossy().contains(".tmp."))
+ .collect();
+ assert!(leftovers.is_empty(), "temp files left: {leftovers:?}");
+ }
+
+ #[test]
+ fn fsync_dir_works() {
+ let dir = tempfile::tempdir().unwrap();
+ fsync_dir(dir.path()).unwrap();
+ }
+}
diff --git a/engine/crates/xerj-common/src/lib.rs b/engine/crates/xerj-common/src/lib.rs
index 7964a1eea..80aebf936 100644
--- a/engine/crates/xerj-common/src/lib.rs
+++ b/engine/crates/xerj-common/src/lib.rs
@@ -21,6 +21,7 @@
pub mod config;
pub mod error;
+pub mod fsio;
pub mod metrics;
pub mod schema;
pub mod types;
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index e6433cfc7..ad4ce5e4c 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -8838,17 +8838,23 @@ impl Index {
}
}
}
- // Force a global WAL checkpoint + rotate + prune at the user-
- // visible flush boundary. `finalize_flush_with_publisher` now
- // time-gates this work (1 s window) on the hot per-shard path;
- // the final flush must bypass the gate so the CLI session's
- // last segment is definitely checkpointed when the CLI exits.
- {
- let seq = self.store.current_seq_no();
- if seq > 0 {
- let _ = self.store.force_wal_maintenance(seq.saturating_sub(1));
- }
- }
+ // Force a global WAL maintenance pass (verified checkpoint +
+ // rotate + prune) at the user-visible flush boundary.
+ // `finalize_flush_with_publisher` time-gates this work (1 s
+ // window) on the hot per-shard path; the final flush must bypass
+ // the gate so the CLI session's last segment is definitely
+ // checkpointed when the CLI exits.
+ //
+ // RC4 W1 #8 — this used to pass `current_seq_no() - 1` as the
+ // checkpoint watermark: the LIVE global counter, covering every
+ // acked-but-unflushed doc in existence (docs bulked into other
+ // shards, docs that landed between a shard's drain and this
+ // call). Maintenance then rotated + pruned their WAL entries
+ // while the docs still lived only in memtables — kill -9 lost
+ // all of them (live repro: 50/50 acked docs gone). The durable
+ // watermark is now derived inside the store from the persisted
+ // snapshot, and prune verifies every entry before destroying it.
+ let _ = self.store.force_wal_maintenance();
// Flushing moves data from memtable → segments, which changes
// what the shortcut count paths compute. Bump the dataset
// version so the response cache invalidates. P3.2: the new
@@ -9404,6 +9410,12 @@ fn pad_with_none<T>(v: &mut Vec<Option<T>>, target_len: usize) {
}
/// Encode columns to bytes and write them to `{segment_id}.dv`.
+///
+/// RC4 W1 #10 — durable write (tmp + fsync + rename + dir fsync): the
+/// `.dv` side-car is part of the segment publish chain whose WAL entries
+/// are pruned ~1 s after the flush; a plain `fs::write` left it in the
+/// volatile page cache, so a power loss after the prune produced a
+/// segment whose doc-values had evaporated.
fn write_doc_values_sidecar(
segments_dir: &std::path::Path,
segment_id: &str,
@@ -9411,7 +9423,7 @@ fn write_doc_values_sidecar(
) -> std::io::Result<()> {
let path = segments_dir.join(format!("{segment_id}.dv"));
let bytes = xerj_storage::doc_values::encode_columns(columns);
- std::fs::write(&path, bytes)
+ xerj_common::fsio::write_file_durable(&path, &bytes)
}
/// Load `{segment_id}.dv`, returning an empty map if the file doesn't
diff --git a/engine/crates/xerj-fts/src/index.rs b/engine/crates/xerj-fts/src/index.rs
index 919c3a266..667c615eb 100644
--- a/engine/crates/xerj-fts/src/index.rs
+++ b/engine/crates/xerj-fts/src/index.rs
@@ -768,7 +768,14 @@ impl FtsIndexWriter {
out.extend_from_slice(&compressed);
out
};
- fs::write(&post_path, &post_bytes_wrapped)
+ // RC4 W1 #10 — every FTS side-car write below goes through the
+ // durable tmp+fsync+rename+dir-fsync pattern. These files are
+ // part of the segment publish chain: the WAL entries they cover
+ // are pruned ~1 s after the flush, so side-cars sitting only in
+ // the volatile page cache meant a power loss could leave a
+ // registered segment with missing/torn FTS data (silently wrong
+ // query results or unreadable fields) with no WAL to recover from.
+ xerj_common::fsio::write_file_durable(&post_path, &post_bytes_wrapped)
.with_context(|| format!("writing postings to {:?}", post_path))?;
// 3. Build and write FST (term → meta byte offset).
@@ -782,8 +789,12 @@ impl FtsIndexWriter {
// inside the meta's flat array — `meta[offset..offset + 24]`
// holds df + ttf + postings_offset + length, so one FST hit →
// one bounded mmap read.
+ // FST streams into a same-directory temp file; fsync + rename +
+ // dir-fsync publishes it durably (RC4 W1 #10 — see the postings
+ // note above).
+ let fst_tmp = segment_dir.join(format!("{}.{}.fst.tmp", segment_id, field_name));
let fst_file = BufWriter::new(
- File::create(&fst_path).with_context(|| format!("creating FST file {:?}", fst_path))?,
+ File::create(&fst_tmp).with_context(|| format!("creating FST file {:?}", fst_tmp))?,
);
let mut fst_builder = MapBuilder::new(fst_file).with_context(|| "creating FST builder")?;
@@ -799,7 +810,18 @@ impl FtsIndexWriter {
.with_context(|| format!("inserting term '{}' into FST", term))?;
}
}
- fst_builder.finish().with_context(|| "finishing FST")?;
+ let mut fst_out = fst_builder
+ .into_inner()
+ .with_context(|| "finishing FST")?;
+ fst_out.flush().with_context(|| "flushing FST")?;
+ fst_out
+ .get_ref()
+ .sync_all()
+ .with_context(|| "fsyncing FST")?;
+ drop(fst_out);
+ fs::rename(&fst_tmp, &fst_path)
+ .with_context(|| format!("publishing FST {:?}", fst_path))?;
+ xerj_common::fsio::fsync_dir(segment_dir).with_context(|| "fsyncing segment dir (fst)")?;
// 4. Write meta in the ZFM4 format — Zstd-19 envelope around
// the per-term records section; ZFM3-compatible header so
@@ -812,7 +834,7 @@ impl FtsIndexWriter {
&sorted_terms,
&term_postings,
)?;
- fs::write(&meta_path, &meta_bytes)
+ xerj_common::fsio::write_file_durable(&meta_path, &meta_bytes)
.with_context(|| format!("writing meta to {:?}", meta_path))?;
// 5. Write norms file — V4 M4.7 compact format.
@@ -852,15 +874,14 @@ impl FtsIndexWriter {
(0, &dense[..])
};
- let norms_file = File::create(&norms_path)
- .with_context(|| format!("creating norms file {:?}", norms_path))?;
- let mut norms_writer = BufWriter::new(norms_file);
- norms_writer.write_all(NORMS_MAGIC)?;
- norms_writer.write_u8(encoding)?;
- norms_writer.write_u32::<LittleEndian>(dense_len as u32)?;
- norms_writer.write_u32::<LittleEndian>(payload.len() as u32)?;
- norms_writer.write_all(payload)?;
- norms_writer.flush()?;
+ let mut norms_bytes: Vec<u8> = Vec::with_capacity(4 + 1 + 4 + 4 + payload.len());
+ norms_bytes.extend_from_slice(NORMS_MAGIC);
+ norms_bytes.push(encoding);
+ norms_bytes.extend_from_slice(&(dense_len as u32).to_le_bytes());
+ norms_bytes.extend_from_slice(&(payload.len() as u32).to_le_bytes());
+ norms_bytes.extend_from_slice(payload);
+ xerj_common::fsio::write_file_durable(&norms_path, &norms_bytes)
+ .with_context(|| format!("writing norms to {:?}", norms_path))?;
Ok(())
}
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 08b0200d1..60b3d8e8a 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -764,7 +764,14 @@ impl IndexStore {
.data_dir
.join("segments")
.join(format!("{segment_id}.ids"));
- std::fs::write(&ids_path, &buf)
+ // RC4 W1 #10 — durable write (tmp + fsync + rename + dir fsync).
+ // The `.ids` side-car is the recovery marker
+ // `recover_orphaned_segments` relies on when a crash lands between
+ // the snapshot publish and the debounced `save_snapshot`; the WAL
+ // entries it supersedes are pruned within ~1 s of the flush. A
+ // plain `fs::write` left it in the page cache — power loss after
+ // the prune GC'd the fully-flushed segment as an orphan.
+ xerj_common::fsio::write_file_durable(&ids_path, &buf)
}
/// Unlink every on-disk file belonging to the given segment ids — the
@@ -1490,39 +1497,24 @@ impl IndexStore {
// mode. Clean shutdown / explicit `_flush` persists via
// `force_wal_maintenance()`.
self.save_snapshot()?;
- // Delete-durability: unpin deletes subsumed by flushed newer
- // versions, then skip maintenance on any shard whose WAL is
- // still the only durable record of an acked delete.
- self.sweep_pending_deletes();
- for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
- let mut wal = ws.lock().unwrap();
- if self.wal_shard_pinned_by_pending_delete(ws_idx) {
- debug!(
- shard = ws_idx,
- "WAL maintenance skipped: shard pinned by unpersisted delete"
- );
- continue;
- }
- wal.checkpoint(max_seq)?;
- // Disk-space fix (2026-07): force-rotate + prune instead of
- // `rotate_if_large(64 MB)`. The checkpoint we just wrote
- // records the shard's FULL current file length (we hold the
- // shard mutex, so `offset == current_offset`), i.e. the
- // whole active generation is covered — rotating now lets
- // `prune()` reclaim it. The old 64 MB threshold never
- // fired on realistic workloads: with 16 WAL shards a 1 M-doc
- // ingest leaves 10-35 MB per shard file, so ~490 MB of WAL
- // that was already durable in segments sat on disk forever
- // (the explicit `_flush` path has done force-rotate via
- // `force_wal_maintenance` since the 2026-04-25 head-to-head;
- // this mirrors it on the background flush path). Rotation
- // churn stays bounded by the surrounding 1 s
- // `WAL_MAINTENANCE_INTERVAL_MS` gate — at most one
- // rotation per shard per second, and `force_rotate` no-ops
- // on generations with no data past the header.
- wal.force_rotate()?;
- wal.prune()?;
- }
+ // RC4 W1 #8 — verified maintenance (see
+ // `wal_maintain_all_verified`). The pre-fix loop here
+ // checkpointed EVERY shard with THIS segment's `max_seq` and
+ // the shard's full file offset, then force-rotated and pruned
+ // — destroying acked-but-unflushed entries on sibling shards
+ // (their memtable docs can carry seqs below a fresh segment's
+ // max) and on THIS shard (docs appended between the drain and
+ // this maintenance tick). Live-verified as the 50/50 kill-9
+ // acked-loss repro. The durable watermark passed for the
+ // compat checkpoint is the max seq registered in the snapshot
+ // we just persisted — never a live counter.
+ //
+ // Disk-space behaviour is preserved: fully-durable generations
+ // still rotate + prune on the same 1 s cadence; generations
+ // holding unproven entries are retained (that retention IS the
+ // fix) and reclaimed on a later tick once their docs flush.
+ let durable_max = self.snapshot.load().max_seq_no;
+ self.wal_maintain_all_verified(durable_max)?;
}
info!(segment_id, doc_count, min_seq, max_seq, "segment flushed");
@@ -1613,10 +1605,7 @@ impl IndexStore {
/// Delete-durability: true if WAL shard `shard_idx` still holds an
/// unpersisted acked delete and therefore MUST NOT be checkpointed,
- /// rotated, or pruned (`WalWriter::prune` deletes any rotated
- /// generation that has a checkpoint file — irrespective of the
- /// checkpoint's `max_seq_no` — so a checkpoint alone already dooms
- /// the Delete entries). Callers hold the shard's WAL mutex when
+ /// rotated, or pruned. Callers hold the shard's WAL mutex when
/// they consult this, which combined with the insert-before-append
/// ordering in [`IndexStore::delete`] makes the check race-free.
fn wal_shard_pinned_by_pending_delete(&self, shard_idx: usize) -> bool {
@@ -1624,61 +1613,127 @@ impl IndexStore {
pending.values().any(|&(_, ws)| ws == shard_idx)
}
- /// Write a checkpoint to ALL WAL shards with the given max_seq_no.
- /// Used by `Index::flush` to write a global checkpoint after
- /// multi-shard parallel flush.
- pub fn wal_checkpoint_all(&self, max_seq: SeqNo) -> Result<()> {
+ /// RC4 W1 #8 — the per-entry durability proof used by WAL maintenance.
+ ///
+ /// Returns true iff destroying this WAL entry cannot lose data:
+ ///
+ /// - `Index`: the doc is tombstoned at an equal-or-newer seq (the
+ /// tombstoning delete is itself WAL-pinned until subsumed, so replay
+ /// reconstructs the deletion); OR a strictly newer version of the doc
+ /// exists (whose own WAL entry is retained until IT is durable — the
+ /// proof chains); OR this exact version has been flushed into a real
+ /// segment (version map repointed off `__memtable__`, which happens
+ /// only after the segment + its side-cars are durably on disk — see
+ /// the blocker-#10 fsync barrier in `finalize_flush_with_publisher`).
+ /// - `Delete`: subsumed — the doc was re-indexed strictly newer AND
+ /// that version is segment-resident (mirrors
+ /// `sweep_pending_deletes`). A load-bearing tombstone is never
+ /// prunable.
+ /// - `UpdateMapping`: always — `schema.json` is written atomically at
+ /// update time and replaying the entry is a no-op.
+ ///
+ /// `None` from the version map is conservatively NOT durable: it can
+ /// mean "version map not yet updated for a just-appended doc" (the
+ /// batch paths append to the WAL before the version-map set).
+ fn wal_entry_durable(&self, entry: &WalEntry, seq: SeqNo) -> bool {
+ match entry {
+ WalEntry::Index { doc_id, .. } => match self.version_map.get(doc_id) {
+ Some(e) if e.deleted => e.seq_no >= seq,
+ Some(e) => {
+ e.seq_no > seq
+ || (e.seq_no == seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID)
+ }
+ None => false,
+ },
+ WalEntry::Delete { doc_id } => match self.version_map.get(doc_id) {
+ Some(e) if !e.deleted => {
+ e.seq_no > seq && &*e.segment_id != IN_MEMORY_SEGMENT_ID
+ }
+ _ => false,
+ },
+ WalEntry::UpdateMapping { .. } => true,
+ }
+ }
+
+ /// RC4 W1 #8 — verified WAL maintenance across all shards.
+ ///
+ /// Replaces the pre-fix `checkpoint(global_max_seq) + force_rotate +
+ /// prune` loop, which destroyed acked-but-unflushed entries two ways:
+ ///
+ /// 1. The checkpoint was written with a GLOBAL max_seq
+ /// (`current_seq_no()-1` from `Index::flush`, or a sibling shard's
+ /// segment max) and the shard's FULL current offset — covering
+ /// entries whose docs still lived only in a memtable. Replay then
+ /// skipped them (loss channel closed by making replay ignore
+ /// checkpoints), and
+ /// 2. `prune()` deleted any rotated generation that had a checkpoint
+ /// file, destroying those same entries outright (loss channel
+ /// closed by per-entry verified pruning).
+ ///
+ /// New per-shard flow (under the shard's WAL mutex):
+ /// - skip entirely if the shard is pinned by an unpersisted delete;
+ /// - decode the ACTIVE generation and check every entry against
+ /// [`wal_entry_durable`](Self::wal_entry_durable);
+ /// - if fully durable: write a checkpoint (safe values — kept for
+ /// data-dir compatibility with older binaries) and force-rotate so
+ /// the generation becomes prunable;
+ /// - if it holds any unproven entry: force-rotate WITHOUT a
+ /// checkpoint (freezes the generation; per-entry verification
+ /// reclaims it on a later tick once everything in it flushed);
+ /// - `prune_verified` every rotated generation.
+ ///
+ /// The caller must persist the snapshot (`save_snapshot`) BEFORE this
+ /// runs so every segment the proofs point at is registered on disk.
+ fn wal_maintain_all_verified(&self, durable_max_seq: SeqNo) -> Result<()> {
self.sweep_pending_deletes();
for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
let mut wal = ws.lock().unwrap();
- // Delete-durability: a checkpoint alone already marks the
- // shard's current generation prunable, so pinned shards must
- // skip it (see `wal_shard_pinned_by_pending_delete`).
if self.wal_shard_pinned_by_pending_delete(ws_idx) {
+ debug!(
+ shard = ws_idx,
+ "WAL maintenance skipped: shard pinned by unpersisted delete"
+ );
continue;
}
- wal.checkpoint(max_seq)?;
+ // Drain the userspace buffer so the on-disk active generation
+ // is complete before we decode it.
+ wal.soft_flush()?;
+ let active_gen = wal.active_generation();
+ let (entries, clean) = wal.read_generation_entries(active_gen);
+ let all_durable =
+ clean && entries.iter().all(|e| self.wal_entry_durable(&e.entry, e.seq_no));
+ if all_durable {
+ wal.checkpoint(durable_max_seq)?;
+ wal.force_rotate()?;
+ } else if !entries.is_empty() {
+ wal.force_rotate()?;
+ }
+ wal.prune_verified(&|entry, seq| self.wal_entry_durable(entry, seq))?;
}
Ok(())
}
- /// Unconditionally run WAL maintenance (checkpoint + force-rotate +
- /// prune) across all shards. Bypasses the
- /// `WAL_MAINTENANCE_INTERVAL_MS` gate that
+ /// Unconditionally run verified WAL maintenance across all shards.
+ /// Bypasses the `WAL_MAINTENANCE_INTERVAL_MS` gate that
/// `finalize_flush_with_publisher` uses on the hot flush path.
/// Called by `Index::flush()` (the final drain / user-triggered
/// `_flush`) so disk cleanup happens immediately — matches ES's
- /// `_flush`-time translog rollover semantics where every user
- /// flush rotates and prunes. The periodic background tick
- /// continues to use the 64 MB threshold (amortises rotation
- /// churn under sustained ingest), but explicit user flushes get
- /// the unconditional rotation so disk gets reclaimed promptly.
- pub fn force_wal_maintenance(&self, max_seq: SeqNo) -> Result<()> {
+ /// `_flush`-time translog rollover semantics.
+ ///
+ /// RC4 W1 #8 — signature change: the old `max_seq: SeqNo` parameter
+ /// is gone. `Index::flush` passed `current_seq_no() - 1`, which
+ /// covered every acked-but-unflushed doc in existence and was the
+ /// direct trigger of the 50/50 kill-9 loss repro. The durable
+ /// watermark is now computed internally from the persisted snapshot
+ /// (max seq actually resident in flushed segments).
+ pub fn force_wal_maintenance(&self) -> Result<()> {
// P2.3 — persist the (possibly debounced) snapshot before pruning
// so an explicit `_flush` / clean shutdown always leaves the
// on-disk snapshot covering every segment whose WAL is about to
// be dropped. Mirrors the coupling in the gated flush path.
self.save_snapshot()?;
- // Delete-durability: same shard-pinning rule as the gated
- // background path — an explicit `_flush`/`_refresh`/shutdown
- // flush must not destroy the WAL Delete entries that are the
- // only durable record of acked deletes (pre-fix this very call,
- // reached from `Index::flush` — including the SIGTERM final
- // flush — pruned them and deleted docs resurrected on restart).
- self.sweep_pending_deletes();
- for (ws_idx, ws) in self.wal_shards.iter().enumerate() {
- let mut wal = ws.lock().unwrap();
- if self.wal_shard_pinned_by_pending_delete(ws_idx) {
- debug!(
- shard = ws_idx,
- "forced WAL maintenance skipped: shard pinned by unpersisted delete"
- );
- continue;
- }
- wal.checkpoint(max_seq)?;
- wal.force_rotate()?;
- wal.prune()?;
- }
+ let durable_max = self.snapshot.load().max_seq_no;
+ self.wal_maintain_all_verified(durable_max)?;
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
@@ -1867,13 +1922,28 @@ impl IndexStore {
std::thread::current().id(),
);
let tmp = path.with_extension(format!("tmp.{nonce}"));
- std::fs::write(&tmp, &bytes)?;
+ // RC4 W1 #10 — the snapshot is the manifest that makes flushed
+ // segments discoverable on restart, and WAL maintenance prunes the
+ // covered entries immediately after this returns. Both the file
+ // bytes and the rename must therefore be durable BEFORE the prune
+ // barrier: write + fsync the tmp, rename, fsync the directory.
+ // Pre-fix (`fs::write` + `rename`, no fsync anywhere) a power loss
+ // within the writeback window could leave an old/absent
+ // snapshot.json next to an already-pruned WAL — flushed segments
+ // then got GC'd as orphans on reopen (acked-data loss).
+ {
+ use std::io::Write as _;
+ let mut f = std::fs::File::create(&tmp)?;
+ f.write_all(&bytes)?;
+ f.sync_all()?;
+ }
// Atomic rename onto the real path. Last writer wins on the
// final snapshot contents, but that's fine: each caller sees
// the same `self.snapshot.load()` atomically-swapped payload
// (arc_swap), so there is no content-level race — only the
// filesystem tmp name was the contention source.
std::fs::rename(&tmp, &path)?;
+ xerj_common::fsio::fsync_dir(&self.data_dir)?;
Ok(())
}
@@ -2564,6 +2634,137 @@ mod tests {
.unwrap()
}
+ /// RC4 W1 #8 regression — bulk-during-flush + kill -9 must lose ZERO
+ /// acked writes.
+ ///
+ /// Live repro this encodes (stream C evidence, 2026-07-12): 50 000-doc
+ /// bulk A, `_flush` dispatched, 50-doc bulk B acked while A's flush
+ /// finalize was in flight, kill -9 after the flush returned, restart →
+ /// `total=50000`, bulk-B survivors **0/50**. Root cause: flush-time
+ /// WAL maintenance checkpointed every shard with a global max_seq
+ /// (`current_seq_no()-1` / the fresh segment's max) + full file
+ /// offset, then `prune()` deleted any checkpointed generation — B's
+ /// WAL entries (memtable-only) were destroyed while B sat in RAM.
+ ///
+ /// In-process kill simulation: `drop(store)` without a flush is
+ /// equivalent to SIGKILL for durability purposes — the memtable (RAM)
+ /// is gone, and the WAL bytes that survive are exactly those the
+ /// appends already pushed to the kernel (every batched append
+ /// soft-flushes, so the userspace buffer is empty at kill time).
+ #[test]
+ fn bulk_during_flush_kill9_loses_zero_acked_writes() {
+ let dir = tempfile::tempdir().unwrap();
+ {
+ let store = open_test_store(dir.path());
+
+ // Bulk A: acked, then drained for a flush (segment write "in
+ // flight" — this is the `_flush` racing point).
+ for i in 0..100 {
+ store
+ .index(format!("a{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ let drained = store.take_memtable_for_flush().unwrap();
+
+ // Bulk B: 50 docs acked AFTER the drain — they miss the
+ // in-flight segment and live only in WAL + memtable.
+ for i in 0..50 {
+ store
+ .index(format!("b{i}"), serde_json::json!({"v": 100000 + i}))
+ .unwrap();
+ }
+
+ // Finalize A's flush — runs the gated WAL maintenance
+ // (pre-fix: checkpoint covering B + rotate + prune = B's WAL
+ // destroyed).
+ store
+ .finalize_flush_with_publisher(drained, |_| Ok(()))
+ .unwrap();
+ // The user-visible flush boundary forces maintenance again
+ // (pre-fix with `current_seq_no() - 1`).
+ store.force_wal_maintenance().unwrap();
+
+ // kill -9.
+ drop(store);
+ }
+
+ // Restart: every acked write must be recoverable.
+ let store2 = open_test_store(dir.path());
+ for i in 0..100 {
+ let e = store2.version_map.get(&format!("a{i}"));
+ assert!(
+ e.map(|e| !e.deleted).unwrap_or(false),
+ "flushed doc a{i} lost after kill+restart"
+ );
+ }
+ let mut lost = Vec::new();
+ for i in 0..50 {
+ let alive = store2
+ .version_map
+ .get(&format!("b{i}"))
+ .map(|e| !e.deleted)
+ .unwrap_or(false);
+ if !alive {
+ lost.push(i);
+ }
+ }
+ assert!(
+ lost.is_empty(),
+ "acked bulk-during-flush docs lost after kill -9: {}/50 (ids {:?})",
+ lost.len(),
+ lost
+ );
+ }
+
+ /// RC4 W1 #8 — after everything is flushed and maintenance runs, the
+ /// verified prune must still reclaim the WAL (no retention leak from
+ /// the new conservatism).
+ #[test]
+ fn verified_prune_still_reclaims_fully_flushed_wal() {
+ let dir = tempfile::tempdir().unwrap();
+ let store = open_test_store(dir.path());
+ for i in 0..100 {
+ store
+ .index(format!("d{i}"), serde_json::json!({"v": i}))
+ .unwrap();
+ }
+ store.flush().unwrap().expect("segment");
+ store.force_wal_maintenance().unwrap();
+
+ // All docs are segment-resident → every generation must be gone
+ // except the fresh empty active one.
+ let wal_dir = store.wal_dir();
+ let mut wal_bytes = 0u64;
+ let mut wal_files = 0usize;
+ for entry in walk_wal_files(&wal_dir) {
+ wal_files += 1;
+ wal_bytes += entry;
+ }
+ // Each surviving file may only be an empty active generation
+ // (16-byte header).
+ assert!(
+ wal_bytes <= wal_files as u64 * 16,
+ "fully-flushed WAL not reclaimed: {wal_files} files, {wal_bytes} bytes"
+ );
+ }
+
+ fn walk_wal_files(root: &Path) -> Vec<u64> {
+ let mut sizes = Vec::new();
+ let mut dirs = vec![root.to_path_buf()];
+ while let Some(d) = dirs.pop() {
+ let Ok(rd) = std::fs::read_dir(&d) else { continue };
+ for e in rd.flatten() {
+ let p = e.path();
+ if p.is_dir() {
+ dirs.push(p);
+ } else if p.extension().map(|x| x == "wal").unwrap_or(false) {
+ sizes.push(e.metadata().map(|m| m.len()).unwrap_or(0));
+ }
+ }
+ }
+ sizes
+ }
+
#[test]
fn index_and_flush() {
let dir = tempfile::tempdir().unwrap();
diff --git a/engine/crates/xerj-storage/src/segment.rs b/engine/crates/xerj-storage/src/segment.rs
index d6c94ce9e..bc40b1f76 100644
--- a/engine/crates/xerj-storage/src/segment.rs
+++ b/engine/crates/xerj-storage/src/segment.rs
@@ -419,6 +419,13 @@ impl SegmentWriter {
}
std::fs::rename(&sidx_tmp, &sidx_path)?;
+ // RC4 W1 #10 — make the renames themselves power-loss durable.
+ // The file CONTENTS were fsynced above, but a rename only becomes
+ // durable when the parent directory is fsynced; without this a
+ // power loss could leave a fully-written segment invisible in the
+ // directory while the WAL entries it covers were already pruned.
+ xerj_common::fsio::fsync_dir(&self.dir)?;
+
debug!(id = %self.id, size = buf.len(), doc_count, "segment written");
Ok(SegmentMeta {
diff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rs
index e7e176084..d3d696162 100644
--- a/engine/crates/xerj-storage/src/wal.rs
+++ b/engine/crates/xerj-storage/src/wal.rs
@@ -24,8 +24,8 @@
//!
//! ## Checkpoint file (`<gen>.wchk`)
//!
-//! Written after a successful segment flush; used to skip already-flushed
-//! entries during replay.
+//! Written after a successful segment flush **only when every entry of the
+//! generation is provably durable** (see the RC4 W1 #8 note below).
//!
//! ```text
//! generation u64
@@ -37,8 +37,33 @@
//! ## Generation rotation
//!
//! When the current WAL file exceeds `wal_max_size_bytes` the writer opens a
-//! new file with `generation + 1`. Old files whose entries are fully covered
-//! by the checkpoint are deleted by [`WalWriter::prune`].
+//! new file with `generation + 1`. Old generations are deleted by
+//! [`WalWriter::prune_verified`] once **every entry they hold** is verified
+//! durable-or-superseded.
+//!
+//! ## Durability contract (RC4 Wave-1 blocker #8)
+//!
+//! A WAL byte may be destroyed (pruned) or ignored (skipped at replay)
+//! **only** if the entry it belongs to is provably durable in a flushed
+//! segment or superseded by a newer durable/replayable version. Two
+//! mechanisms enforce this:
+//!
+//! 1. **Replay replays everything.** Checkpoints are no longer used to
+//! skip entries at replay: both replay consumers (the storage memtable
+//! rebuild and the engine FTS memtable rebuild) are idempotent — they
+//! check the segment-rebuilt version map and skip entries whose doc is
+//! already segment-resident at an equal-or-newer seq. The pre-fix skip
+//! rule (`positionally-covered && seq_no <= checkpoint.max_seq_no`)
+//! silently discarded acked-but-unflushed entries whenever a checkpoint
+//! over-covered — e.g. the flush path checkpointing
+//! `current_seq_no()-1` while other shards' memtables still held acked
+//! docs with smaller seqs. Live-verified as 50/50 acked-doc loss on
+//! kill -9 racing `_flush` (RC4 review, stream C).
+//!
+//! 2. **Prune verifies per entry.** [`WalWriter::prune_verified`] decodes
+//! every entry of a candidate generation and deletes the file only if
+//! the caller-supplied verifier proves every single entry durable. A
+//! generation holding even one acked-but-unflushed entry is retained.
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufReader, BufWriter, Read, Seek, SeekFrom, Write};
@@ -567,60 +592,112 @@ impl WalWriter {
Ok(())
}
- /// Remove WAL generations whose entries are fully covered by a checkpoint.
+ /// Current active generation number.
+ pub fn active_generation(&self) -> u64 {
+ self.generation
+ }
+
+ /// Read every decodable entry of generation `gen` from disk.
+ ///
+ /// Returns `(entries, clean)` where `clean == true` means the file was
+ /// decoded to the end without a single framing/CRC/JSON error. Callers
+ /// verifying prunability MUST treat `clean == false` as "cannot prove
+ /// anything about the undecodable tail" and retain the generation.
///
- /// V4 M4 — extended semantics: any generation file with `gen < self.generation`
- /// that has its own checkpoint (`gen.wchk`) is safe to delete because the
- /// checkpoint guarantees all its entries are already durable in segments.
- /// This lets `finalize_flush_with_publisher → rotate_if_large → prune`
- /// actually reclaim space instead of keeping old WAL files indefinitely.
- pub fn prune(&self) -> Result<()> {
+ /// For the ACTIVE generation the caller must hold the WAL mutex (this
+ /// is a `&self` read of an append-only file; the writer soft-flushes
+ /// on every append so the on-disk prefix is complete up to
+ /// `current_offset`).
+ pub fn read_generation_entries(&self, gen: u64) -> (Vec<ReplayEntry>, bool) {
+ let path = wal_path(&self.dir, gen);
+ if !path.exists() {
+ return (Vec::new(), true);
+ }
+ let results = read_wal_file(path);
+ let clean = results.iter().all(|r| r.is_ok());
+ let entries = results.into_iter().filter_map(|r| r.ok()).collect();
+ (entries, clean)
+ }
+
+ /// RC4 W1 #8 — remove WAL generations whose entries are **all verified
+ /// durable-or-superseded** by the caller-supplied predicate.
+ ///
+ /// This replaces the pre-fix `prune()` whose rule was "gen < active &&
+ /// gen has a checkpoint file". That rule destroyed acked-but-unflushed
+ /// entries: the flush path checkpointed every shard with a *global*
+ /// max_seq (`current_seq_no()-1`, or a sibling shard's segment max)
+ /// and a full-file offset, so a generation still holding entries whose
+ /// docs lived only in the memtable got a checkpoint and was deleted —
+ /// kill -9 then lost every one of those acked docs (live-verified
+ /// 50/50 loss).
+ ///
+ /// New rule: decode every entry of every `gen < active_generation` and
+ /// delete the file only when
+ /// 1. the file decodes cleanly end-to-end (no torn/corrupt frames), and
+ /// 2. `verify(entry, seq_no)` returns true for EVERY entry.
+ ///
+ /// The verifier is supplied by `IndexStore` and proves an entry durable
+ /// via the version map (doc segment-resident at `>=` seq, superseded by
+ /// a newer retained version, or tombstoned by a retained delete).
+ ///
+ /// ENOENT on `remove_file` is benign: sharded flushes run in parallel
+ /// and every one of them prunes ALL WAL shards; two concurrent
+ /// maintenance passes can race to delete the same file — the loser
+ /// gets NotFound.
+ ///
+ /// Returns the number of generations pruned.
+ pub fn prune_verified(
+ &self,
+ verify: &dyn Fn(&WalEntry, SeqNo) -> bool,
+ ) -> Result<usize> {
let active_gen = self.generation;
let entries = fs::read_dir(&self.dir)?;
- // Collect (gen, path, is_wal_file, is_checkpoint_file) tuples.
let mut wal_files: Vec<(u64, std::path::PathBuf)> = Vec::new();
- let mut chkpt_files: std::collections::HashSet<u64> = std::collections::HashSet::new();
for entry in entries.flatten() {
let name = entry.file_name();
let name_str = name.to_string_lossy();
if name_str.ends_with(".wal") {
if let Some(gen) = parse_wal_generation(&name_str) {
- wal_files.push((gen, entry.path()));
- }
- } else if let Some(stem) = name_str.strip_suffix(".wchk") {
- if let Ok(gen) = u64::from_str_radix(stem, 16) {
- chkpt_files.insert(gen);
+ if gen < active_gen {
+ wal_files.push((gen, entry.path()));
+ }
}
}
}
- // Delete every .wal file whose generation is strictly older than
- // the active generation AND has its own checkpoint. Also delete
- // the stale checkpoint files once their corresponding wal is gone.
- //
- // ENOENT on `remove_file` is benign: sharded flushes run in
- // parallel and every one of them calls `prune()` across ALL WAL
- // shards (see `finalize_flush_with_publisher`). Two concurrent
- // flushes can both see the same old-generation .wal file in
- // `read_dir` and race to delete it; the loser gets NotFound.
- // Pre-fix, that loser's whole flush aborted with "No such file"
- // and its shard's data was left in the memtable for the next
- // tick — measurable as ~1/5 shard-flushes failing on sustained
- // 20 M ingest.
+ let mut pruned = 0usize;
for (gen, path) in &wal_files {
- if *gen < active_gen && chkpt_files.contains(gen) {
- match fs::remove_file(path) {
- Ok(()) => debug!(gen, "pruned WAL file"),
- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
- debug!(gen, "WAL file already pruned by sibling flush — ok");
- }
- Err(e) => return Err(e.into()),
+ let (gen_entries, clean) = self.read_generation_entries(*gen);
+ if !clean {
+ // Undecodable bytes — cannot prove them durable; keep the
+ // generation. (Torn tails from a previous crash park here
+ // until the Wave-2 torn-tail truncation lands.)
+ debug!(gen, "WAL generation retained: undecodable entries");
+ continue;
+ }
+ let all_durable = gen_entries.iter().all(|e| verify(&e.entry, e.seq_no));
+ if !all_durable {
+ debug!(
+ gen,
+ entries = gen_entries.len(),
+ "WAL generation retained: holds acked-but-unflushed entries"
+ );
+ continue;
+ }
+ match fs::remove_file(path) {
+ Ok(()) => {
+ pruned += 1;
+ debug!(gen, entries = gen_entries.len(), "pruned WAL generation");
+ }
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+ debug!(gen, "WAL file already pruned by sibling flush — ok");
}
- let ck = checkpoint_path(&self.dir, *gen);
- let _ = fs::remove_file(&ck);
+ Err(e) => return Err(e.into()),
}
+ let ck = checkpoint_path(&self.dir, *gen);
+ let _ = fs::remove_file(&ck);
}
- Ok(())
+ Ok(pruned)
}
fn rotate(&mut self) -> Result<()> {
@@ -748,44 +825,34 @@ impl WalReader {
}
}
- /// Return an iterator of all entries that are NOT yet covered by a
- /// checkpoint, ordered by sequence number.
+ /// Return an iterator over **every** entry in every surviving WAL
+ /// generation, ordered by sequence number within each file.
+ ///
+ /// RC4 W1 #8 — replay no longer consults checkpoints to skip entries.
+ /// Both replay consumers (`IndexStore::replay_wal` and the engine FTS
+ /// memtable rebuild) are idempotent: they skip entries whose doc is
+ /// already segment-resident at an equal-or-newer seq via the
+ /// segment-rebuilt version map. Skipping at the WAL layer, by
+ /// contrast, silently DISCARDED acked-but-unflushed entries whenever a
+ /// checkpoint over-covered (stale checkpoints, global max_seq written
+ /// onto per-shard files, full-file offsets recorded while sibling
+ /// memtables still held acked docs). Surviving generations are exactly
+ /// the not-yet-verified-durable set (see `prune_verified`), so the
+ /// replay cost of reading them in full is proportional to the data
+ /// that genuinely needs recovery.
pub fn replay(&self) -> Result<impl Iterator<Item = Result<ReplayEntry>> + '_> {
- let (latest_gen, _) = find_latest_generation(&self.dir)?;
-
- // Determine the starting generation and offset from the checkpoint
- let checkpoint = match read_checkpoint(&self.dir, latest_gen) {
- Ok(c) => {
- // debug: fires per WAL (16 shards × every index) at startup.
- debug!(
- generation = c.generation,
- max_seq_no = c.max_seq_no,
- offset = c.offset,
- "replaying from checkpoint"
- );
- Some(c)
- }
- Err(_) => {
- debug!("no checkpoint found, replaying from generation 0");
- None
- }
- };
- let start_gen = checkpoint.as_ref().map(|c| c.generation).unwrap_or(0);
-
- // Collect all generations >= start_gen
let mut gens: Vec<u64> = fs::read_dir(&self.dir)
.into_iter()
.flatten()
.flatten()
.filter_map(|e| parse_wal_generation(&e.file_name().to_string_lossy()))
- .filter(|&g| g >= start_gen)
.collect();
gens.sort_unstable();
let dir = self.dir.clone();
let iter = gens.into_iter().flat_map(move |gen| {
let path = wal_path(&dir, gen);
- read_wal_file(path, checkpoint)
+ read_wal_file(path)
});
Ok(iter)
@@ -854,22 +921,16 @@ pub fn replay_all_sorted(wal_root: &Path) -> Vec<ReplayEntry> {
all_entries
}
-/// Read all entries from a single WAL file, skipping those already covered
-/// by `checkpoint`.
+/// Read all entries from a single WAL file.
///
-/// OFFSET-BOUNDED SKIP (2026-07, S3 hardening): an entry is "covered" only
-/// when it sits BEFORE the checkpointed byte offset of the checkpointed
-/// generation (or in an older generation). A checkpoint by construction
-/// covers exactly the bytes `[0, checkpoint.offset)` of its generation —
-/// anything appended past that offset was written AFTER the checkpoint and
-/// must replay regardless of its seq_no. The old rule (`seq_no <=
-/// checkpoint.max_seq_no`, applied file-wide) silently discarded every
-/// post-restart acked op whenever the seq counter had regressed across a
-/// restart (stale .wchk with max_seq_no=X on the active generation, tail
-/// entries seq 1..K <= X) — 100% loss of the acked tail. It also protects
-/// against the flush-checkpoint race where a reserved-but-unflushed seq
-/// range is covered by a concurrent checkpoint's max_seq.
-fn read_wal_file(path: PathBuf, checkpoint: Option<WalCheckpoint>) -> Vec<Result<ReplayEntry>> {
+/// RC4 W1 #8: checkpoint-based skipping is GONE from replay (see
+/// `WalReader::replay` docs). The 2026-07 S3 "offset-bounded skip"
+/// hardening was the previous, weaker containment of the same bug class
+/// (stale/over-broad checkpoints discarding acked entries); removing the
+/// skip entirely is its logical conclusion — replay-side loss is now
+/// structurally impossible, and dedup of already-flushed entries is the
+/// job of the idempotent replay consumers.
+fn read_wal_file(path: PathBuf) -> Vec<Result<ReplayEntry>> {
let file = match File::open(&path) {
Ok(f) => f,
Err(e) => return vec![Err(e.into())],
@@ -957,16 +1018,6 @@ fn read_wal_file(path: PathBuf, checkpoint: Option<WalCheckpoint>) -> Vec<Result
let this_offset = file_offset;
file_offset += entry_total;
- // Skip entries already covered by the checkpoint — see the
- // offset-bounded rule in the function docs.
- if let Some(c) = &checkpoint {
- let positionally_covered =
- generation < c.generation || (generation == c.generation && this_offset < c.offset);
- if positionally_covered && seq_no <= c.max_seq_no {
- continue;
- }
- }
-
// Check and strip the compression flag from op_code.
let is_compressed = (op_code & OP_COMPRESSED_FLAG) != 0;
let raw_op_code = op_code & !OP_COMPRESSED_FLAG;
@@ -1123,6 +1174,11 @@ fn create_wal_file(path: &Path, generation: u64) -> io::Result<File> {
OpenOptions::new().append(true).open(path)
}
+/// Read a generation's checkpoint file. Replay no longer consumes
+/// checkpoints (RC4 W1 #8); they are still WRITTEN — with verified-safe
+/// values only — for data-dir compatibility with older binaries whose
+/// replay/prune still read them. Kept for tests and offline tooling.
+#[allow(dead_code)]
fn read_checkpoint(dir: &Path, generation: u64) -> io::Result<WalCheckpoint> {
let path = checkpoint_path(dir, generation);
let mut f = File::open(&path)?;
@@ -1180,7 +1236,13 @@ mod tests {
}
#[test]
- fn checkpoint_skips_old_entries() {
+ fn replay_ignores_checkpoints_and_returns_everything() {
+ // RC4 W1 #8: checkpoints must never cause replay to discard
+ // entries. Pre-fix, a checkpoint with max_seq_no=3 made replay
+ // skip seqs 1..3 — if any of those were acked-but-unflushed
+ // (over-broad checkpoint), they were silently lost. Replay now
+ // returns every surviving entry; dedup of genuinely-flushed docs
+ // is the idempotent replay consumers' job (version-map guarded).
let dir = tempfile::tempdir().unwrap();
let seq_ctr = Arc::new(AtomicU64::new(1));
let mut w = WalWriter::open(
@@ -1197,15 +1259,66 @@ mod tests {
})
.unwrap();
}
- // Checkpoint after seq 3
+ // A (deliberately over-broad) checkpoint after seq 3.
w.checkpoint(3).unwrap();
drop(w);
let reader = WalReader::new(dir.path());
let entries: Vec<_> = reader.replay().unwrap().collect();
- // Only seq 4 and 5 should be returned (seq_no=4,5 since we started at 1)
- assert!(entries.iter().all(|e| e.as_ref().unwrap().seq_no > 3));
- assert_eq!(entries.len(), 2);
+ assert_eq!(entries.len(), 5, "replay must return ALL entries");
+ }
+
+ #[test]
+ fn prune_verified_retains_generations_with_unflushed_entries() {
+ // RC4 W1 #8 core regression at the WAL layer: a rotated
+ // generation holding even one entry the verifier cannot prove
+ // durable must survive prune; fully-verified generations are
+ // deleted.
+ let dir = tempfile::tempdir().unwrap();
+ let seq_ctr = Arc::new(AtomicU64::new(1));
+ let mut w = WalWriter::open(
+ dir.path(),
+ 64 * 1024 * 1024,
+ SyncMode::Batched,
+ Arc::clone(&seq_ctr),
+ )
+ .unwrap();
+
+ // Gen 0: seqs 1..=3. Rotate. Gen 1: seqs 4..=5. Rotate.
+ for i in 0..3 {
+ w.append(&WalEntry::Delete {
+ doc_id: format!("g0-{i}"),
+ })
+ .unwrap();
+ }
+ w.force_rotate().unwrap();
+ for i in 0..2 {
+ w.append(&WalEntry::Delete {
+ doc_id: format!("g1-{i}"),
+ })
+ .unwrap();
+ }
+ w.force_rotate().unwrap();
+
+ // Verifier: everything with seq <= 3 is durable; seq 4+ is not
+ // (i.e. gen 1 holds acked-but-unflushed entries).
+ let pruned = w.prune_verified(&|_e, seq| seq <= 3).unwrap();
+ assert_eq!(pruned, 1, "exactly gen 0 must be pruned");
+
+ // Gen 1's entries must still replay in full.
+ let reader = WalReader::new(dir.path());
+ let survivors: Vec<_> = reader
+ .replay()
+ .unwrap()
+ .map(|r| r.unwrap().seq_no)
+ .collect();
+ assert_eq!(survivors, vec![4, 5], "unverified generation retained");
+
+ // Once the verifier can prove everything, the rest is reclaimed.
+ let pruned2 = w.prune_verified(&|_e, _seq| true).unwrap();
+ assert_eq!(pruned2, 1);
+ let reader = WalReader::new(dir.path());
+ assert_eq!(reader.replay().unwrap().count(), 0);
}
#[test]