Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(storage): honor wal_sync="sync" on bulk paths + implement the wal_batch_ms fsync loop (RC4 W1 #9)

xerj-org085f0f4f747f
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 79e57b7a1..9121ef616 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -142,10 +142,15 @@ impl Config {
}
}
- // Storage: WAL batch interval sanity
+ // Storage: WAL batch interval sanity (documented range: 1..=10000).
if self.storage.wal_batch_ms == 0 {
return Err(XerjError::config("storage.wal_batch_ms must be > 0"));
}
+ if self.storage.wal_batch_ms > 10_000 {
+ return Err(XerjError::config(
+ "storage.wal_batch_ms must be <= 10000 (10 s)",
+ ));
+ }
// Merge: min_segments must be >= 2
if self.merge.min_segments < 2 {
@@ -322,11 +327,21 @@ pub struct TlsConfig {
pub struct StorageConfig {
/// WAL fsync strategy: `"sync"`, `"batched"`, or `"async"` (default: `"batched"`).
///
- /// - `"sync"` — fsync after every write. Safest, slowest.
- /// - `"batched"` — fsync every `wal_batch_ms`. Good balance.
- /// - `"async"` — never fsync. Fastest, least durable.
+ /// - `"sync"` — fsync before every acknowledgement. On the bulk API
+ /// this is one fsync per bulk request (group commit), matching ES's
+ /// per-request translog fsync granularity. Safest, slowest.
+ /// - `"batched"` — writes reach the kernel immediately (process-crash
+ /// durable); a background loop fsyncs every `wal_batch_ms`, bounding
+ /// the power-loss window. Good balance.
+ /// - `"async"` — never fsync; the OS decides when to write back.
+ /// Fastest, least durable.
+ ///
+ /// RC4 W1 #9: `"sync"` was previously ignored on the bulk ingest paths
+ /// (fsync only via the undocumented `XERJ_STRICT_SYNC` env var) and
+ /// the `wal_batch_ms` loop did not exist. Both are honored now.
pub wal_sync: WalSync,
/// How often to fsync the WAL when `wal_sync = "batched"` (default: `100` ms).
+ /// Range: 1..=10000.
pub wal_batch_ms: u64,
/// Maximum WAL file size before it is rolled over (default: `512` MiB).
pub wal_max_size_mb: u64,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index ad4ce5e4c..ae78eeeeb 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -15028,10 +15028,19 @@ fn store_config_from(config: &Config) -> IndexStoreConfig {
SyncMode::Batched
}
};
+ // RC4 W1 #9 — plumb the fsync cadence for the batched-mode background
+ // fsync loop. Only `wal_sync = "batched"` gets the loop: "sync"
+ // fsyncs inline per request, "async" is the explicit never-fsync
+ // opt-out (both are expressed as wal_batch_ms = 0 here).
+ let wal_batch_ms = match config.storage.wal_sync {
+ xerj_common::config::WalSync::Batched => config.storage.wal_batch_ms,
+ xerj_common::config::WalSync::Sync | xerj_common::config::WalSync::Async => 0,
+ };
IndexStoreConfig {
memtable_max_bytes: (config.storage.flush_size_mb * 1024 * 1024) as usize,
wal_max_size_bytes: config.storage.wal_max_size_mb * 1024 * 1024,
sync_mode,
+ wal_batch_ms,
schema_version: 1,
storage_mode: xerj_storage::StorageMode::Local,
num_wal_shards: config.engine.ingest_shards,
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 60b3d8e8a..2302ffa97 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -240,7 +240,26 @@ pub struct IndexStoreConfig {
/// Maximum WAL file size before rotation.
pub wal_max_size_bytes: u64,
/// WAL sync mode.
+ ///
+ /// RC4 W1 #9 — this is now honored EVERYWHERE, including the bulk
+ /// paths (`wal_append_batch` / `wal_append_batch_raw`), which
+ /// previously forced Batched behaviour and only fsynced via the
+ /// `XERJ_STRICT_SYNC` env var — silently ignoring an operator's
+ /// explicit `wal_sync = "sync"` opt-in.
+ ///
+ /// - `Strict` (`wal_sync = "sync"`): fsync before every ack. On the
+ /// bulk paths this is one fsync per bulk request (group commit) —
+ /// the same granularity as ES's per-request translog fsync.
+ /// - `Batched` (`wal_sync = "batched"`): writes reach the kernel page
+ /// cache before ack (process-crash durable); a background loop
+ /// fsyncs every dirty WAL shard every `wal_batch_ms` (power-loss
+ /// window bounded to `wal_batch_ms`).
pub sync_mode: SyncMode,
+ /// RC4 W1 #9 — fsync cadence (milliseconds) of the background WAL
+ /// fsync loop when `sync_mode == Batched`. `0` disables the loop
+ /// (used by `wal_sync = "async"`: never fsync, OS decides — and by
+ /// unit tests that don't want the thread).
+ pub wal_batch_ms: u64,
/// Schema version for new segments.
pub schema_version: u32,
/// Storage destination for flushed segments.
@@ -257,6 +276,7 @@ impl Default for IndexStoreConfig {
memtable_max_bytes: 32 * 1024 * 1024, // 32 MiB
wal_max_size_bytes: 128 * 1024 * 1024, // 128 MiB
sync_mode: SyncMode::Batched,
+ wal_batch_ms: 0,
schema_version: 1,
storage_mode: StorageMode::Local,
num_wal_shards: 1,
@@ -264,6 +284,26 @@ impl Default for IndexStoreConfig {
}
}
+/// RC4 W1 #9 — one-time loud warning when `XERJ_SKIP_WAL` disables the
+/// write-ahead log entirely. Pre-fix the env var was honored silently;
+/// an operator (or stray benchmark script) could run a production node
+/// with ZERO durability and nothing in the logs saying so.
+fn warn_skip_wal_once() {
+ static ONCE: std::sync::Once = std::sync::Once::new();
+ ONCE.call_once(|| {
+ warn!(
+ "XERJ_SKIP_WAL is set: the write-ahead log is DISABLED. \
+ Acknowledged writes exist only in memory until a segment \
+ flush — ANY crash loses them. Never use this outside \
+ throwaway benchmarks."
+ );
+ eprintln!(
+ "WARNING: XERJ_SKIP_WAL is set — WAL disabled, acked writes are \
+ NOT crash-durable. Never use this outside throwaway benchmarks."
+ );
+ });
+}
+
// ── IndexStore ────────────────────────────────────────────────────────────────
/// Per-index storage engine.
@@ -512,6 +552,35 @@ impl IndexStore {
// Replay WAL to rebuild in-memory state (these override segment entries).
store.replay_wal(&wal_dir)?;
+ // RC4 W1 #9 — the `wal_batch_ms` fsync loop. The config has
+ // documented `wal_sync = "batched"` as "fsync every wal_batch_ms"
+ // since 0.x, but nothing implemented it: the only fsyncs happened
+ // at flush/rotate boundaries, so the real power-loss window was
+ // unbounded (up to `flush_interval_secs`). A detached thread now
+ // fsyncs every DIRTY WAL shard on the configured cadence. It
+ // holds only a Weak ref — the loop exits within one tick of the
+ // store being dropped. Strict mode fsyncs inline per request and
+ // Async mode opts out of fsync entirely; neither spawns the loop
+ // (wal_batch_ms is forced to 0 for them by `store_config_from`).
+ if store.config.sync_mode == SyncMode::Batched && store.config.wal_batch_ms > 0 {
+ let weak: std::sync::Weak<IndexStore> = Arc::downgrade(&store);
+ let period = std::time::Duration::from_millis(store.config.wal_batch_ms);
+ let _ = std::thread::Builder::new()
+ .name("xerj-wal-fsync".into())
+ .spawn(move || loop {
+ std::thread::sleep(period);
+ let Some(s) = weak.upgrade() else { break };
+ for shard in &s.wal_shards {
+ let mut wal = shard.lock().unwrap();
+ if wal.is_dirty() {
+ if let Err(e) = wal.sync() {
+ warn!("wal_batch_ms fsync failed: {e}");
+ }
+ }
+ }
+ });
+ }
+
info!(data_dir = ?data_dir, "IndexStore opened");
Ok(store)
}
@@ -2305,6 +2374,7 @@ impl IndexStore {
}
if std::env::var("XERJ_SKIP_WAL").is_ok() {
+ warn_skip_wal_once();
let n = docs.len() as u64;
let start_seq = self
.seq_counter
@@ -2384,13 +2454,21 @@ impl IndexStore {
{
let ws = self.wal_shard_for(&docs[0].0);
let mut wal = self.wal_lock_shard(ws);
+ // Suppress the writer's per-append fsync while the pre-framed
+ // batch is emitted, then issue at most ONE sync for the whole
+ // batch below (group commit).
let saved_mode = wal.sync_mode();
wal.set_sync_mode(SyncMode::Batched);
wal.append_frames_locked(&frames, total_written)?;
- let strict_sync = std::env::var("XERJ_STRICT_SYNC")
- .map(|v| !v.is_empty() && v != "0")
- .unwrap_or(false);
- let sync_result = if strict_sync {
+ // RC4 W1 #9 — honor the operator's configured durability.
+ // Pre-fix this path fsynced ONLY when the undocumented
+ // XERJ_STRICT_SYNC env var was set; `wal_sync = "sync"` in the
+ // config was silently ignored on every bulk request.
+ let strict = self.config.sync_mode == SyncMode::Strict
+ || std::env::var("XERJ_STRICT_SYNC")
+ .map(|v| !v.is_empty() && v != "0")
+ .unwrap_or(false);
+ let sync_result = if strict {
wal.sync()
} else {
wal.soft_flush()
@@ -2445,6 +2523,7 @@ impl IndexStore {
// only do CRC32 + a single `write_all` of the pre-framed
// batch buffer.
if std::env::var("XERJ_SKIP_WAL").is_ok() {
+ warn_skip_wal_once();
let n = docs.len() as u64;
let start_seq = self
.seq_counter
@@ -2548,32 +2627,24 @@ impl IndexStore {
let saved_mode = wal.sync_mode();
wal.set_sync_mode(SyncMode::Batched);
wal.append_frames_locked(&frames, total_written)?;
- // M5.4 — SKIP the per-batch fsync on the bulk hot path.
- //
- // Pre-M5.4 each bulk batch called `wal.sync()` which
- // issues a `fsync(2)` — on NVMe that's ~1 ms of wall
- // time. At 76 batches/sec (380 k/s ingest) that's
- // 76 ms/sec of fsync blocking = ~8 % of the ingest
- // wall time, spent purely on filesystem round-trips.
+ // M5.4 — skip the per-batch fsync on the DEFAULT (batched)
+ // bulk hot path: each fsync(2) is ~1 ms on NVMe, ~8 % of
+ // ingest wall time at 76 batches/s. Without it the WAL
+ // bytes are in the kernel page cache — a process crash
+ // loses nothing; power loss is bounded by the `wal_batch_ms`
+ // background fsync loop (RC4 W1 #9).
//
- // Durability trade-off: without per-batch fsync, the
- // WAL bytes are buffered in the kernel page cache. A
- // process crash loses nothing (the kernel flushes on
- // close). A kernel crash / power loss between batches
- // can lose the last N seconds of data — same window as
- // a segment flush interval anyway. The WAL still does
- // a periodic `sync` at rotation boundaries and at flush
- // time, and the segment commit path calls `fsync` on
- // the finished .seg file before the snapshot swap, so
- // acknowledged-and-flushed data is never lost.
- //
- // Setting `SyncMode::NoSync` via the `XERJ_STRICT_SYNC`
- // env var re-enables per-batch fsync for workloads
- // that need it.
- let strict_sync = std::env::var("XERJ_STRICT_SYNC")
- .map(|v| !v.is_empty() && v != "0")
- .unwrap_or(false);
- let sync_result = if strict_sync {
+ // RC4 W1 #9 — `wal_sync = "sync"` (SyncMode::Strict) is now
+ // HONORED here: one fsync per bulk request before the ack
+ // (group commit — the same granularity as ES's per-request
+ // translog fsync). Pre-fix the config was silently ignored
+ // and only the undocumented XERJ_STRICT_SYNC env var (kept
+ // as an override) forced the fsync.
+ let strict = self.config.sync_mode == SyncMode::Strict
+ || std::env::var("XERJ_STRICT_SYNC")
+ .map(|v| !v.is_empty() && v != "0")
+ .unwrap_or(false);
+ let sync_result = if strict {
wal.sync()
} else {
// Just flush the BufWriter to the kernel, skip
diff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rs
index d3d696162..635fe0888 100644
--- a/engine/crates/xerj-storage/src/wal.rs
+++ b/engine/crates/xerj-storage/src/wal.rs
@@ -225,6 +225,10 @@ pub struct WalWriter {
current_offset: u64,
max_size_bytes: u64,
sync_mode: SyncMode,
+ /// True when bytes were appended since the last `fsync` — consumed by
+ /// the `wal_batch_ms` background fsync loop (RC4 W1 #9) so idle shards
+ /// don't get pointless fsyncs.
+ dirty: bool,
/// Shared sequence number counter — also used by the index store.
pub seq_counter: Arc<AtomicU64>,
}
@@ -266,6 +270,7 @@ impl WalWriter {
current_offset,
max_size_bytes,
sync_mode,
+ dirty: false,
seq_counter,
})
}
@@ -329,6 +334,7 @@ impl WalWriter {
let written = 4 + 8 + 1 + payload.len() as u64 + 4;
self.current_offset += written;
+ self.dirty = true;
if self.sync_mode == SyncMode::Strict {
self.sync()?;
@@ -428,9 +434,16 @@ impl WalWriter {
let written = 4 + 8 + 1 + payload.len() as u64 + 4;
self.current_offset += written;
+ self.dirty = true;
if self.sync_mode == SyncMode::Strict {
self.sync()?;
+ } else {
+ // Durability consistency with `append` (RC4 W1 #8/#9): drain
+ // the BufWriter to the kernel page cache before the caller
+ // acks, so a SIGKILL cannot lose an acked single-doc append
+ // that was still sitting in the userspace buffer.
+ self.writer.flush()?;
}
Ok(seq_no)
}
@@ -447,6 +460,7 @@ impl WalWriter {
}
self.writer.write_all(frames)?;
self.current_offset += total_written;
+ self.dirty = true;
if self.sync_mode == SyncMode::Strict {
self.sync()?;
}
@@ -527,6 +541,7 @@ impl WalWriter {
// Single write_all — one BufWriter call for the whole batch.
self.writer.write_all(&out)?;
self.current_offset += written_total;
+ self.dirty = true;
if self.sync_mode == SyncMode::Strict {
self.sync()?;
@@ -538,9 +553,17 @@ impl WalWriter {
pub fn sync(&mut self) -> Result<()> {
self.writer.flush()?;
self.writer.get_ref().sync_all()?;
+ self.dirty = false;
Ok(())
}
+ /// True when bytes were appended since the last `fsync`. Consumed by
+ /// the `wal_batch_ms` background fsync loop so idle shards are not
+ /// pointlessly fsynced (RC4 W1 #9).
+ pub fn is_dirty(&self) -> bool {
+ self.dirty
+ }
+
/// Flush the BufWriter to the kernel without a `fsync(2)`.
///
/// The buffered bytes are visible to any reader on the same
diff --git a/engine/xerj.default.toml b/engine/xerj.default.toml
index be65dc05e..d2f92393a 100644
--- a/engine/xerj.default.toml
+++ b/engine/xerj.default.toml
@@ -76,12 +76,17 @@ key_path = ""
# WAL fsync strategy. Controls the durability/throughput trade-off.
#
-# "sync" — fsync after every write. Maximum durability; lowest throughput.
-# Use for financial or compliance workloads.
-# "batched" — fsync every wal_batch_ms milliseconds. Good balance.
+# "sync" — fsync before every acknowledgement (one fsync per bulk
+# request — group commit, same granularity as Elasticsearch's
+# per-request translog fsync). Maximum durability; lowest
+# throughput. Use for financial or compliance workloads.
+# "batched" — writes reach the kernel before the ack (process-crash
+# durable); a background loop fsyncs every wal_batch_ms,
+# bounding the power-loss window to wal_batch_ms.
# Default and recommended for most workloads.
# "async" — never fsync; OS decides when to flush. Maximum throughput.
-# Risk of losing up to ~1 second of data on crash.
+# Power loss can lose un-flushed data; a process crash alone
+# loses nothing (writes are in the kernel page cache).
wal_sync = "batched"
# How often to fsync the WAL when wal_sync = "batched" (milliseconds).