Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(storage): bound per-shard WAL buffers to 64 KiB

buger761b91507550
diff --git a/engine/crates/xerj-storage/src/index_store.rs b/engine/crates/xerj-storage/src/index_store.rs
index 63c1f04c8..5051ac054 100644
--- a/engine/crates/xerj-storage/src/index_store.rs
+++ b/engine/crates/xerj-storage/src/index_store.rs
@@ -3214,6 +3214,44 @@ impl IndexStore {
mod tests {
use super::*;
+ /// Structural bound: 16 empty indices × 8 WAL shards each must retain
+ /// exactly `128 × 64 KiB` of userspace buffer capacity — not the old
+ /// `128 × 8 MiB` reservation. Asserts `buffer_capacity()` (deterministic)
+ /// rather than process RSS.
+ #[test]
+ fn empty_multi_index_wal_buffers_have_a_bounded_total_capacity() {
+ let root = tempfile::tempdir().unwrap();
+ let mut stores = Vec::new();
+ let mut total_capacity = 0usize;
+
+ for index in 0..16 {
+ let store = IndexStore::open(
+ root.path().join(format!("index-{index}")),
+ IndexStoreConfig {
+ sync_mode: SyncMode::Batched,
+ wal_batch_ms: 0,
+ num_wal_shards: 8,
+ ..Default::default()
+ },
+ )
+ .unwrap();
+ assert_eq!(store.wal_shards.len(), 8);
+ total_capacity += store
+ .wal_shards
+ .iter()
+ .map(|writer| writer.lock().unwrap().buffer_capacity())
+ .sum::<usize>();
+ stores.push(store);
+ }
+
+ assert_eq!(stores.len(), 16);
+ assert_eq!(total_capacity, 16 * 8 * 64 * 1024);
+ assert!(
+ total_capacity <= 8 * 1024 * 1024,
+ "128 idle WAL shards retained {total_capacity} bytes of buffers"
+ );
+ }
+
fn open_test_store(dir: &Path) -> Arc<IndexStore> {
IndexStore::open(
dir,
diff --git a/engine/crates/xerj-storage/src/wal.rs b/engine/crates/xerj-storage/src/wal.rs
index 46ca23952..8edee5087 100644
--- a/engine/crates/xerj-storage/src/wal.rs
+++ b/engine/crates/xerj-storage/src/wal.rs
@@ -95,7 +95,17 @@ const WAL_FRAME_OVERHEAD: usize = 17;
const WAL_MAX_ENTRY_LEN: u32 = 1 << 30; // 1 GiB
/// BufWriter capacity shared by open/rotate/recovery reseats.
-const WAL_BUF_CAP: usize = 8 * 1024 * 1024;
+///
+/// Every append drains this userspace buffer before acknowledging the write,
+/// so a multi-megabyte capacity cannot batch across requests. Keeping it at
+/// a conventional I/O-buffer size bounds idle retention when every index owns
+/// multiple WAL shards (the old 8 MiB capacity retained 1 GiB for 128 shards).
+///
+/// Capacity is an allocation/coalescing bound only: frame payloads may be
+/// larger than `WAL_BUF_CAP` (up to `WAL_MAX_ENTRY_LEN`). Rust's `BufWriter`
+/// flushes and issues direct writes for oversized `write_all` slices, so
+/// correctness of large records is independent of this constant.
+const WAL_BUF_CAP: usize = 64 * 1024;
// Op codes
const OP_INDEX: u8 = 0x01;
@@ -328,6 +338,15 @@ pub struct WalWriter {
}
impl WalWriter {
+ /// Userspace `BufWriter` capacity for this writer (test/instrumentation).
+ ///
+ /// Always equals [`WAL_BUF_CAP`] after open, rotate, truncate-reseat, and
+ /// fresh-generation recovery. Does not grow with record size.
+ #[cfg(test)]
+ pub(crate) fn buffer_capacity(&self) -> usize {
+ self.writer.capacity()
+ }
+
/// Open (or create) the WAL in `dir`.
///
/// If a WAL file for the latest generation already exists it is opened for
@@ -1835,6 +1854,11 @@ mod tests {
source: serde_json::json!({"v": 2, "pad": "x".repeat(64)}),
});
assert!(err.is_err(), "append during ENOSPC must NACK");
+ assert_eq!(
+ w.buffer_capacity(),
+ WAL_BUF_CAP,
+ "truncate-and-reseat recovery must preserve the bounded capacity"
+ );
// Space frees up (recovery reseated the writer on a clean fd, so
// the fault is naturally gone) — C is acked.
@@ -1907,6 +1931,27 @@ mod tests {
assert!(clean);
assert_eq!(entries.len(), 2, "only a and c survive: {entries:?}");
assert_eq!(entries[1].seq_no, seq_c);
+ assert_eq!(
+ w.buffer_capacity(),
+ WAL_BUF_CAP,
+ "batch error recovery must preserve the bounded capacity"
+ );
+ drop(w);
+
+ // Batched mode flushes each acknowledged append into the kernel even
+ // though it does not fsync inline. Reopen must retain every acked
+ // frame and use the same bounded writer capacity.
+ let reopened =
+ WalWriter::open(dir.path(), 64 * 1024 * 1024, SyncMode::Batched, seq_ctr).unwrap();
+ assert_eq!(reopened.buffer_capacity(), WAL_BUF_CAP);
+ drop(reopened);
+ let replayed: Vec<_> = WalReader::new(dir.path())
+ .replay()
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ assert_eq!(replayed.len(), 2);
+ assert_eq!(replayed[1].seq_no, seq_c);
}
/// Crash-torn tail heal at open: a partial frame left at the file
@@ -2070,4 +2115,204 @@ mod tests {
let entries: Vec<_> = reader.replay().unwrap().map(|r| r.unwrap()).collect();
assert_eq!(entries.len(), 3);
}
+
+ #[test]
+ fn bounded_buffer_survives_append_rotate_and_reopen() {
+ let dir = tempfile::tempdir().unwrap();
+ let seq_counter = Arc::new(AtomicU64::new(1));
+ let mut writer = WalWriter::open(
+ dir.path(),
+ 64 * 1024 * 1024,
+ SyncMode::Strict,
+ Arc::clone(&seq_counter),
+ )
+ .unwrap();
+ assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+
+ let first = WalEntry::Index {
+ doc_id: "before-rotate".into(),
+ source: serde_json::json!({"payload": "durable"}),
+ };
+ assert_eq!(writer.append(&first).unwrap(), 1);
+ writer.rotate().unwrap();
+ assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+
+ let second = WalEntry::Index {
+ doc_id: "after-rotate".into(),
+ source: serde_json::json!({"payload": "also durable"}),
+ };
+ assert_eq!(writer.append(&second).unwrap(), 2);
+ drop(writer);
+
+ let reopened =
+ WalWriter::open(dir.path(), 64 * 1024 * 1024, SyncMode::Strict, seq_counter).unwrap();
+ assert_eq!(reopened.buffer_capacity(), WAL_BUF_CAP);
+ drop(reopened);
+
+ let entries: Vec<_> = WalReader::new(dir.path())
+ .replay()
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ assert_eq!(entries.len(), 2);
+ assert_eq!(entries[0].seq_no, 1);
+ assert_eq!(entries[1].seq_no, 2);
+ }
+
+ /// Records smaller than, equal to, and much larger than `WAL_BUF_CAP`
+ /// must all survive append, forced rotation (truncation of generations),
+ /// reopen, and exact replay. Capacity stays fixed regardless of payload.
+ #[test]
+ fn records_across_buffer_capacity_survive_append_rotate_reopen() {
+ let dir = tempfile::tempdir().unwrap();
+ let seq_counter = Arc::new(AtomicU64::new(1));
+ let mut writer = WalWriter::open(
+ dir.path(),
+ // Force rotation after a few large frames.
+ 256 * 1024,
+ SyncMode::Batched,
+ Arc::clone(&seq_counter),
+ )
+ .unwrap();
+ assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+
+ // High-entropy printable so LZ4 cannot collapse oversized payloads
+ // back under the buffer capacity.
+ let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+ let mk_payload = |n: usize, salt: u32| {
+ let mut state = salt;
+ let body: String = (0..n)
+ .map(|_| {
+ state ^= state << 13;
+ state ^= state >> 17;
+ state ^= state << 5;
+ alphabet[state as usize % alphabet.len()] as char
+ })
+ .collect();
+ serde_json::json!({"body": body, "n": n})
+ };
+
+ let shapes: &[(usize, &str)] = &[
+ (64, "small"),
+ (WAL_BUF_CAP, "equal-cap"),
+ (WAL_BUF_CAP * 2 + 17, "2x-cap"),
+ (131_072, "131kib"),
+ ];
+ let mut expected_ids = Vec::new();
+ for (size, label) in shapes {
+ for copy in 0..3 {
+ let doc_id = format!("{label}-{copy}");
+ let seq = writer
+ .append(&WalEntry::Index {
+ doc_id: doc_id.clone(),
+ source: mk_payload(*size, (size + copy) as u32 + 0x9e37_79b9),
+ })
+ .unwrap();
+ assert!(seq > 0);
+ expected_ids.push(doc_id);
+ assert_eq!(
+ writer.buffer_capacity(),
+ WAL_BUF_CAP,
+ "capacity must not grow with record size {size}"
+ );
+ }
+ }
+ writer.force_rotate().unwrap();
+ assert_eq!(writer.buffer_capacity(), WAL_BUF_CAP);
+
+ // One more append after rotate to prove the new generation is live.
+ let final_id = "post-rotate".to_string();
+ writer
+ .append(&WalEntry::Index {
+ doc_id: final_id.clone(),
+ source: mk_payload(128, 7),
+ })
+ .unwrap();
+ expected_ids.push(final_id);
+ drop(writer);
+
+ let reopened = WalWriter::open(
+ dir.path(),
+ 256 * 1024,
+ SyncMode::Batched,
+ Arc::clone(&seq_counter),
+ )
+ .unwrap();
+ assert_eq!(reopened.buffer_capacity(), WAL_BUF_CAP);
+ drop(reopened);
+
+ let replayed: Vec<_> = WalReader::new(dir.path())
+ .replay()
+ .unwrap()
+ .map(|entry| entry.unwrap())
+ .collect();
+ let ids: Vec<String> = replayed
+ .iter()
+ .map(|e| match &e.entry {
+ WalEntry::Index { doc_id, .. } => doc_id.clone(),
+ other => panic!("unexpected entry {other:?}"),
+ })
+ .collect();
+ assert_eq!(ids, expected_ids);
+ }
+
+ /// Concurrent shard writers sharing one sequence counter must keep the
+ /// bounded capacity and produce an exact combined replay set.
+ #[test]
+ fn concurrent_shard_writers_preserve_capacity_and_exact_replay() {
+ use std::sync::Barrier;
+ use std::thread;
+
+ let root = tempfile::tempdir().unwrap();
+ let seq = Arc::new(AtomicU64::new(1));
+ let barrier = Arc::new(Barrier::new(5));
+ let mut handles = Vec::new();
+ for shard in 0..4 {
+ let dir = root.path().join(format!("s{shard}"));
+ fs::create_dir_all(&dir).unwrap();
+ let seq = Arc::clone(&seq);
+ let barrier = Arc::clone(&barrier);
+ handles.push(thread::spawn(move || {
+ let mut w =
+ WalWriter::open(&dir, 64 * 1024 * 1024, SyncMode::Batched, seq).unwrap();
+ assert_eq!(w.buffer_capacity(), WAL_BUF_CAP);
+ barrier.wait();
+ for i in 0..50 {
+ let pad = if i % 10 == 0 {
+ "X".repeat(WAL_BUF_CAP + 256)
+ } else {
+ "y".repeat(32)
+ };
+ w.append(&WalEntry::Index {
+ doc_id: format!("s{shard}-{i}"),
+ source: serde_json::json!({"pad": pad}),
+ })
+ .unwrap();
+ }
+ w.sync().unwrap();
+ assert_eq!(w.buffer_capacity(), WAL_BUF_CAP);
+ }));
+ }
+ barrier.wait();
+ for h in handles {
+ h.join().unwrap();
+ }
+
+ let mut total = 0usize;
+ for shard in 0..4 {
+ let dir = root.path().join(format!("s{shard}"));
+ let replayed: Vec<_> = WalReader::new(&dir)
+ .replay()
+ .unwrap()
+ .map(|e| e.unwrap())
+ .collect();
+ assert_eq!(replayed.len(), 50);
+ total += replayed.len();
+ let reopened =
+ WalWriter::open(&dir, 64 * 1024 * 1024, SyncMode::Batched, Arc::clone(&seq))
+ .unwrap();
+ assert_eq!(reopened.buffer_capacity(), WAL_BUF_CAP);
+ }
+ assert_eq!(total, 200);
+ }
}