AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(storage): honor wal_sync="sync" on bulk paths + implement the wal_batch_ms fsync loop (RC4 W1 #9)
xerj-org085f0f4f747f
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex 79e57b7a1..9121ef616 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -142,10 +142,15 @@ impl Config {}}- // Storage: WAL batch interval sanity+ // Storage: WAL batch interval sanity (documented range: 1..=10000).if self.storage.wal_batch_ms == 0 {return Err(XerjError::config("storage.wal_batch_ms must be > 0"));}+ if self.storage.wal_batch_ms > 10_000 {+ return Err(XerjError::config(+ "storage.wal_batch_ms must be <= 10000 (10 s)",+ ));+ }// Merge: min_segments must be >= 2if self.merge.min_segments < 2 {@@ -322,11 +327,21 @@ pub struct TlsConfig {pub struct StorageConfig {/// WAL fsync strategy: `"sync"`, `"batched"`, or `"async"` (default: `"batched"`).///- /// - `"sync"` — fsync after every write. Safest, slowest.- /// - `"batched"` — fsync every `wal_batch_ms`. Good balance.- /// - `"async"` — never fsync. Fastest, least durable.+ /// - `"sync"` — fsync before every acknowledgement. On the bulk API+ /// this is one fsync per bulk request (group commit), matching ES's+ /// per-request translog fsync granularity. Safest, slowest.+ /// - `"batched"` — writes reach the kernel immediately (process-crash+ /// durable); a background loop fsyncs every `wal_batch_ms`, bounding+ /// the power-loss window. Good balance.+ /// - `"async"` — never fsync; the OS decides when to write back.+ /// Fastest, least durable.+ ///+ /// RC4 W1 #9: `"sync"` was previously ignored on the bulk ingest paths+ /// (fsync only via the undocumented `XERJ_STRICT_SYNC` env var) and+ /// the `wal_batch_ms` loop did not exist. Both are honored now.pub wal_sync: WalSync,/// How often to fsync the WAL when `wal_sync = "batched"` (default: `100` ms).+ /// Range: 1..=10000.pub wal_batch_ms: u64,/// Maximum WAL file size before it is rolled over (default: `512` MiB).pub wal_max_size_mb: u64,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex ad4ce5e4c..ae78eeeeb 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -15028,10 +15028,19 @@ fn store_config_from(config: &Config) -> IndexStoreConfig {SyncMode::Batched}};+ // RC4 W1 #9 — plumb the fsync cadence for the batched-mode background+ // fsync loop. Only `wal_sync = "batched"` gets the loop: "sync"+ // fsyncs inline per request, "async" is the explicit never-fsync+ // opt-out (both are expressed as wal_batch_ms = 0 here).+ let wal_batch_ms = match config.storage.wal_sync {+ xerj_common::config::WalSync::Batched => config.storage.wal_batch_ms,+ xerj_common::config::WalSync::Sync | xerj_common::config::WalSync::Async => 0,+ };IndexStoreConfig {memtable_max_bytes: (config.storage.flush_size_mb * 1024 * 1024) as usize,wal_max_size_bytes: config.storage.wal_max_size_mb * 1024 * 1024,sync_mode,+ wal_batch_ms,schema_version: 1,storage_mode: xerj_storage::StorageMode::Local,num_wal_shards: config.engine.ingest_shards,diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rsindex 60b3d8e8a..2302ffa97 100644--- a/engine/crates/xerj-storage/src/index_store.rs+++ b/engine/crates/xerj-storage/src/index_store.rs@@ -240,7 +240,26 @@ pub struct IndexStoreConfig {/// Maximum WAL file size before rotation.pub wal_max_size_bytes: u64,/// WAL sync mode.+ ///+ /// RC4 W1 #9 — this is now honored EVERYWHERE, including the bulk+ /// paths (`wal_append_batch` / `wal_append_batch_raw`), which+ /// previously forced Batched behaviour and only fsynced via the+ /// `XERJ_STRICT_SYNC` env var — silently ignoring an operator's+ /// explicit `wal_sync = "sync"` opt-in.+ ///+ /// - `Strict` (`wal_sync = "sync"`): fsync before every ack. On the+ /// bulk paths this is one fsync per bulk request (group commit) —+ /// the same granularity as ES's per-request translog fsync.+ /// - `Batched` (`wal_sync = "batched"`): writes reach the kernel page+ /// cache before ack (process-crash durable); a background loop+ /// fsyncs every dirty WAL shard every `wal_batch_ms` (power-loss+ /// window bounded to `wal_batch_ms`).pub sync_mode: SyncMode,+ /// RC4 W1 #9 — fsync cadence (milliseconds) of the background WAL+ /// fsync loop when `sync_mode == Batched`. `0` disables the loop+ /// (used by `wal_sync = "async"`: never fsync, OS decides — and by+ /// unit tests that don't want the thread).+ pub wal_batch_ms: u64,/// Schema version for new segments.pub schema_version: u32,/// Storage destination for flushed segments.@@ -257,6 +276,7 @@ impl Default for IndexStoreConfig {memtable_max_bytes: 32 * 1024 * 1024, // 32 MiBwal_max_size_bytes: 128 * 1024 * 1024, // 128 MiBsync_mode: SyncMode::Batched,+ wal_batch_ms: 0,schema_version: 1,storage_mode: StorageMode::Local,num_wal_shards: 1,@@ -264,6 +284,26 @@ impl Default for IndexStoreConfig {}}+/// RC4 W1 #9 — one-time loud warning when `XERJ_SKIP_WAL` disables the+/// write-ahead log entirely. Pre-fix the env var was honored silently;+/// an operator (or stray benchmark script) could run a production node+/// with ZERO durability and nothing in the logs saying so.+fn warn_skip_wal_once() {+ static ONCE: std::sync::Once = std::sync::Once::new();+ ONCE.call_once(|| {+ warn!(+ "XERJ_SKIP_WAL is set: the write-ahead log is DISABLED. \+ Acknowledged writes exist only in memory until a segment \+ flush — ANY crash loses them. Never use this outside \+ throwaway benchmarks."+ );+ eprintln!(+ "WARNING: XERJ_SKIP_WAL is set — WAL disabled, acked writes are \+ NOT crash-durable. Never use this outside throwaway benchmarks."+ );+ });+}+// ── IndexStore ────────────────────────────────────────────────────────────────/// Per-index storage engine.@@ -512,6 +552,35 @@ impl IndexStore {// Replay WAL to rebuild in-memory state (these override segment entries).store.replay_wal(&wal_dir)?;+ // RC4 W1 #9 — the `wal_batch_ms` fsync loop. The config has+ // documented `wal_sync = "batched"` as "fsync every wal_batch_ms"+ // since 0.x, but nothing implemented it: the only fsyncs happened+ // at flush/rotate boundaries, so the real power-loss window was+ // unbounded (up to `flush_interval_secs`). A detached thread now+ // fsyncs every DIRTY WAL shard on the configured cadence. It+ // holds only a Weak ref — the loop exits within one tick of the+ // store being dropped. Strict mode fsyncs inline per request and+ // Async mode opts out of fsync entirely; neither spawns the loop+ // (wal_batch_ms is forced to 0 for them by `store_config_from`).+ if store.config.sync_mode == SyncMode::Batched && store.config.wal_batch_ms > 0 {+ let weak: std::sync::Weak<IndexStore> = Arc::downgrade(&store);+ let period = std::time::Duration::from_millis(store.config.wal_batch_ms);+ let _ = std::thread::Builder::new()+ .name("xerj-wal-fsync".into())+ .spawn(move || loop {+ std::thread::sleep(period);+ let Some(s) = weak.upgrade() else { break };+ for shard in &s.wal_shards {+ let mut wal = shard.lock().unwrap();+ if wal.is_dirty() {+ if let Err(e) = wal.sync() {+ warn!("wal_batch_ms fsync failed: {e}");+ }+ }+ }+ });+ }+info!(data_dir = ?data_dir, "IndexStore opened");Ok(store)}@@ -2305,6 +2374,7 @@ impl IndexStore {}if std::env::var("XERJ_SKIP_WAL").is_ok() {+ warn_skip_wal_once();let n = docs.len() as u64;let start_seq = self.seq_counter@@ -2384,13 +2454,21 @@ impl IndexStore {{let ws = self.wal_shard_for(&docs[0].0);let mut wal = self.wal_lock_shard(ws);+ // Suppress the writer's per-append fsync while the pre-framed+ // batch is emitted, then issue at most ONE sync for the whole+ // batch below (group commit).let saved_mode = wal.sync_mode();wal.set_sync_mode(SyncMode::Batched);wal.append_frames_locked(&frames, total_written)?;- let strict_sync = std::env::var("XERJ_STRICT_SYNC")- .map(|v| !v.is_empty() && v != "0")- .unwrap_or(false);- let sync_result = if strict_sync {+ // RC4 W1 #9 — honor the operator's configured durability.+ // Pre-fix this path fsynced ONLY when the undocumented+ // XERJ_STRICT_SYNC env var was set; `wal_sync = "sync"` in the+ // config was silently ignored on every bulk request.+ let strict = self.config.sync_mode == SyncMode::Strict+ || std::env::var("XERJ_STRICT_SYNC")+ .map(|v| !v.is_empty() && v != "0")+ .unwrap_or(false);+ let sync_result = if strict {wal.sync()} else {wal.soft_flush()@@ -2445,6 +2523,7 @@ impl IndexStore {// only do CRC32 + a single `write_all` of the pre-framed// batch buffer.if std::env::var("XERJ_SKIP_WAL").is_ok() {+ warn_skip_wal_once();let n = docs.len() as u64;let start_seq = self.seq_counter@@ -2548,32 +2627,24 @@ impl IndexStore {let saved_mode = wal.sync_mode();wal.set_sync_mode(SyncMode::Batched);wal.append_frames_locked(&frames, total_written)?;- // M5.4 — SKIP the per-batch fsync on the bulk hot path.- //- // Pre-M5.4 each bulk batch called `wal.sync()` which- // issues a `fsync(2)` — on NVMe that's ~1 ms of wall- // time. At 76 batches/sec (380 k/s ingest) that's- // 76 ms/sec of fsync blocking = ~8 % of the ingest- // wall time, spent purely on filesystem round-trips.+ // M5.4 — skip the per-batch fsync on the DEFAULT (batched)+ // bulk hot path: each fsync(2) is ~1 ms on NVMe, ~8 % of+ // ingest wall time at 76 batches/s. Without it the WAL+ // bytes are in the kernel page cache — a process crash+ // loses nothing; power loss is bounded by the `wal_batch_ms`+ // background fsync loop (RC4 W1 #9).//- // Durability trade-off: without per-batch fsync, the- // WAL bytes are buffered in the kernel page cache. A- // process crash loses nothing (the kernel flushes on- // close). A kernel crash / power loss between batches- // can lose the last N seconds of data — same window as- // a segment flush interval anyway. The WAL still does- // a periodic `sync` at rotation boundaries and at flush- // time, and the segment commit path calls `fsync` on- // the finished .seg file before the snapshot swap, so- // acknowledged-and-flushed data is never lost.- //- // Setting `SyncMode::NoSync` via the `XERJ_STRICT_SYNC`- // env var re-enables per-batch fsync for workloads- // that need it.- let strict_sync = std::env::var("XERJ_STRICT_SYNC")- .map(|v| !v.is_empty() && v != "0")- .unwrap_or(false);- let sync_result = if strict_sync {+ // RC4 W1 #9 — `wal_sync = "sync"` (SyncMode::Strict) is now+ // HONORED here: one fsync per bulk request before the ack+ // (group commit — the same granularity as ES's per-request+ // translog fsync). Pre-fix the config was silently ignored+ // and only the undocumented XERJ_STRICT_SYNC env var (kept+ // as an override) forced the fsync.+ let strict = self.config.sync_mode == SyncMode::Strict+ || std::env::var("XERJ_STRICT_SYNC")+ .map(|v| !v.is_empty() && v != "0")+ .unwrap_or(false);+ let sync_result = if strict {wal.sync()} else {// Just flush the BufWriter to the kernel, skipdiff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rsindex d3d696162..635fe0888 100644--- a/engine/crates/xerj-storage/src/wal.rs+++ b/engine/crates/xerj-storage/src/wal.rs@@ -225,6 +225,10 @@ pub struct WalWriter {current_offset: u64,max_size_bytes: u64,sync_mode: SyncMode,+ /// True when bytes were appended since the last `fsync` — consumed by+ /// the `wal_batch_ms` background fsync loop (RC4 W1 #9) so idle shards+ /// don't get pointless fsyncs.+ dirty: bool,/// Shared sequence number counter — also used by the index store.pub seq_counter: Arc<AtomicU64>,}@@ -266,6 +270,7 @@ impl WalWriter {current_offset,max_size_bytes,sync_mode,+ dirty: false,seq_counter,})}@@ -329,6 +334,7 @@ impl WalWriter {let written = 4 + 8 + 1 + payload.len() as u64 + 4;self.current_offset += written;+ self.dirty = true;if self.sync_mode == SyncMode::Strict {self.sync()?;@@ -428,9 +434,16 @@ impl WalWriter {let written = 4 + 8 + 1 + payload.len() as u64 + 4;self.current_offset += written;+ self.dirty = true;if self.sync_mode == SyncMode::Strict {self.sync()?;+ } else {+ // Durability consistency with `append` (RC4 W1 #8/#9): drain+ // the BufWriter to the kernel page cache before the caller+ // acks, so a SIGKILL cannot lose an acked single-doc append+ // that was still sitting in the userspace buffer.+ self.writer.flush()?;}Ok(seq_no)}@@ -447,6 +460,7 @@ impl WalWriter {}self.writer.write_all(frames)?;self.current_offset += total_written;+ self.dirty = true;if self.sync_mode == SyncMode::Strict {self.sync()?;}@@ -527,6 +541,7 @@ impl WalWriter {// Single write_all — one BufWriter call for the whole batch.self.writer.write_all(&out)?;self.current_offset += written_total;+ self.dirty = true;if self.sync_mode == SyncMode::Strict {self.sync()?;@@ -538,9 +553,17 @@ impl WalWriter {pub fn sync(&mut self) -> Result<()> {self.writer.flush()?;self.writer.get_ref().sync_all()?;+ self.dirty = false;Ok(())}+ /// True when bytes were appended since the last `fsync`. Consumed by+ /// the `wal_batch_ms` background fsync loop so idle shards are not+ /// pointlessly fsynced (RC4 W1 #9).+ pub fn is_dirty(&self) -> bool {+ self.dirty+ }+/// Flush the BufWriter to the kernel without a `fsync(2)`.////// The buffered bytes are visible to any reader on the samediff --git a/engine/xerj.default.toml b/engine/xerj.default.tomlindex be65dc05e..d2f92393a 100644--- a/engine/xerj.default.toml+++ b/engine/xerj.default.toml@@ -76,12 +76,17 @@ key_path = ""# WAL fsync strategy. Controls the durability/throughput trade-off.#-# "sync" — fsync after every write. Maximum durability; lowest throughput.-# Use for financial or compliance workloads.-# "batched" — fsync every wal_batch_ms milliseconds. Good balance.+# "sync" — fsync before every acknowledgement (one fsync per bulk+# request — group commit, same granularity as Elasticsearch's+# per-request translog fsync). Maximum durability; lowest+# throughput. Use for financial or compliance workloads.+# "batched" — writes reach the kernel before the ack (process-crash+# durable); a background loop fsyncs every wal_batch_ms,+# bounding the power-loss window to wal_batch_ms.# Default and recommended for most workloads.# "async" — never fsync; OS decides when to flush. Maximum throughput.-# Risk of losing up to ~1 second of data on crash.+# Power loss can lose un-flushed data; a process crash alone+# loses nothing (writes are in the kernel page cache).wal_sync = "batched"# How often to fsync the WAL when wal_sync = "batched" (milliseconds).