AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(wal-shards): gate the per-index WAL shard override that 5f26032 shipped untested
xerj-org0fd3287ca626
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex b9f95b939..bfa2519e3 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -21458,7 +21458,8 @@ fn parse_passage_guard(ids: &Value) -> Option<HashSet<String>> {/// file descriptor instead of one per ingest shard — the ingest-shard count/// scales with CPU cores, so N indices held ~N×cores WAL fds open and exhausted/// the (low, ~256) macOS file-descriptor limit at a few hundred datasets.-/// Absent => the global `engine.ingest_shards` default (backward compatible).+/// Absent, or out of `1..=MAX_PER_INDEX_WAL_SHARDS` => the global+/// `engine.ingest_shards` default (backward compatible).////// MUST be persisted at create and honored on open so the WAL *write* layout/// stays consistent (doc routing is `xxh3(id) & (num_shards-1)`). Replay itself@@ -21469,10 +21470,19 @@ fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {settings.pointer("/index/xerj_ingest_shards").and_then(Value::as_u64)- .filter(|&n| n >= 1)+ .filter(|&n| (1..=MAX_PER_INDEX_WAL_SHARDS).contains(&n)).map(|n| n as usize)}+/// Ceiling for the per-index override, mirroring the `engine.ingest_shards`+/// limit enforced by `EngineConfig::validate`. The global setting is validated+/// at startup; this one arrives in an index-create request body (es_compat+/// threads `settings` through untouched) and `IndexStore::open` eagerly creates+/// a directory and opens a WAL file descriptor per shard, so an unbounded value+/// is a create-time fd exhaustion — the exact failure this setting exists to+/// prevent.+const MAX_PER_INDEX_WAL_SHARDS: u64 = 256;+fn store_config_from(config: &Config, wal_shards_override: Option<usize>) -> IndexStoreConfig {let sync_mode = match config.storage.wal_sync {xerj_common::config::WalSync::Sync => SyncMode::Strict,@@ -32512,3 +32522,9 @@ mod flush_memory_integration_tests {// its own file (src/graph.rs) instead of growing this one.#[path = "graph.rs"]pub mod graph;++// Child module (same `#[path]` reason as `graph`): the WAL-shard override+// helpers are private free functions of this module.+#[cfg(test)]+#[path = "wal_shard_settings_tests.rs"]+mod wal_shard_settings_tests;diff --git a/engine/crates/xerj-engine/src/wal_shard_settings_tests.rs b/engine/crates/xerj-engine/src/wal_shard_settings_tests.rsnew file mode 100644index 000000000..e2b74c97a--- /dev/null+++ b/engine/crates/xerj-engine/src/wal_shard_settings_tests.rs@@ -0,0 +1,262 @@+//! Regressions for the per-index WAL shard override (`index.xerj_ingest_shards`).+//!+//! The override exists because each index eagerly opens one WAL file descriptor+//! per ingest shard, and the ingest-shard count scales with CPU cores: a few+//! hundred autoindex datasets exhausted the macOS fd limit. Two properties have+//! to hold or the fix is worse than the bug — the value must be *ignored* unless+//! it is a sane count, and it must be honored identically at create and on+//! reopen, since the WAL write layout (`wal/*.wal` vs `wal/s{N}/*.wal`) and doc+//! routing both derive from it.++use std::path::Path;++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_common::types::Schema;++use super::{store_config_from, wal_shards_override_from_settings};+use crate::Engine;++fn settings_with(shards: Value) -> Value {+ json!({ "index": { "xerj_ingest_shards": shards } })+}++/// The `s{N}` shard subdirectories under an index's WAL root. Zero of them means+/// the single-shard (legacy root) layout; `IndexStore::open` creates one per+/// shard whenever the count is > 1, so this is the on-disk witness of the shard+/// count the store was actually opened with.+fn wal_shard_dirs(index_dir: &Path) -> usize {+ std::fs::read_dir(index_dir.join("wal"))+ .expect("wal dir")+ .flatten()+ .filter(|e| {+ e.path().is_dir()+ && e.file_name()+ .to_string_lossy()+ .strip_prefix('s')+ .is_some_and(|n| n.parse::<usize>().is_ok())+ })+ .count()+}++fn root_wal_files(index_dir: &Path) -> usize {+ std::fs::read_dir(index_dir.join("wal"))+ .expect("wal dir")+ .flatten()+ .filter(|e| e.path().extension().is_some_and(|ext| ext == "wal"))+ .count()+}++/// `engine.ingest_shards` defaults to a fraction of the host's core count, so a+/// test that let it default would silently assert nothing on a 1- or 2-core+/// runner (the default would already be 1). Pin it well above 1.+fn config_for(dir: &TempDir) -> Config {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.engine.ingest_shards = 8;+ config+}++#[test]+fn a_positive_wal_shard_count_in_index_settings_is_used() {+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(json!(1))),+ Some(1)+ );+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(json!(4))),+ Some(4)+ );+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(json!(256))),+ Some(256),+ "256 is the documented ceiling and must still be accepted"+ );+}++#[test]+fn an_absent_wal_shard_setting_leaves_the_engine_default_in_charge() {+ for absent in [+ Value::Null,+ json!({}),+ json!({ "index": {} }),+ json!({ "index": { "number_of_shards": 1 } }),+ json!({ "xerj_ingest_shards": 1 }),+ json!({ "index.xerj_ingest_shards": 1 }),+ ] {+ assert_eq!(+ wal_shards_override_from_settings(&absent),+ None,+ "no override should be read from {absent}"+ );+ }+}++#[test]+fn zero_and_negative_wal_shard_counts_are_ignored_never_used_as_a_count() {+ for bad in [json!(0), json!(-1), json!(-8)] {+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(bad.clone())),+ None,+ "{bad} must not become a shard count"+ );+ }+}++#[test]+fn non_numeric_wal_shard_values_are_ignored_never_coerced() {+ for bad in [+ json!("4"),+ json!("many"),+ json!(4.5),+ json!(true),+ json!([4]),+ json!({ "value": 4 }),+ Value::Null,+ ] {+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(bad.clone())),+ None,+ "{bad} must not become a shard count"+ );+ }+}++/// The value arrives straight off an index-create request body, and every shard+/// costs a directory plus a permanently open WAL file descriptor at+/// `IndexStore::open`. An out-of-range count is refused outright rather than+/// clamped, so a typo falls back to the engine default instead of quietly+/// running with a shard count nobody asked for.+#[test]+fn absurdly_large_wal_shard_counts_are_refused_and_never_opened() {+ for absurd in [json!(257), json!(100_000), json!(u64::MAX)] {+ assert_eq!(+ wal_shards_override_from_settings(&settings_with(absurd.clone())),+ None,+ "{absurd} exceeds the 256 ceiling and must be refused"+ );+ }+}++#[test]+fn the_store_config_takes_its_wal_shard_count_from_the_override_when_present() {+ let dir = TempDir::new().expect("temp dir");+ let config = config_for(&dir);++ assert_eq!(store_config_from(&config, None).num_wal_shards, 8);+ assert_eq!(store_config_from(&config, Some(1)).num_wal_shards, 1);+ assert_eq!(store_config_from(&config, Some(2)).num_wal_shards, 2);+}++/// The whole point of persisting the setting: create and reopen must agree on+/// the shard count. If reopen fell back to `engine.ingest_shards`, the reopened+/// store would write into `wal/s{N}/` while the create-time entries live in+/// `wal/`, splitting the layout under a single index.+#[tokio::test]+async fn a_pinned_wal_shard_count_survives_reopen_with_every_document_intact() {+ let dir = TempDir::new().expect("temp dir");+ let config = config_for(&dir);+ let pinned_dir = dir.path().join("pinned");+ let default_dir = dir.path().join("spread");++ // Deliberately below the flush thresholds: the documents stay in the+ // memtable + WAL, so surviving a reopen means the WAL was replayed out of+ // the layout the persisted shard count selected, not read back from a+ // segment that would have survived either way.+ async fn write_docs(idx: &crate::Index, range: std::ops::Range<u32>) {+ for i in range {+ idx.index_document(+ Some(format!("doc-{i}")),+ json!({ "body": format!("payload {i}") }),+ )+ .await+ .expect("index document");+ }+ }++ async fn assert_docs(idx: &crate::Index, count: u32) {+ for i in 0..count {+ let doc = idx+ .get_document(&format!("doc-{i}"))+ .await+ .expect("get document")+ .unwrap_or_else(|| panic!("doc-{i} was lost"));+ assert_eq!(doc["body"], json!(format!("payload {i}")));+ }+ let all = idx+ .search(+ &xerj_query::parse_request(&json!({ "query": { "match_all": {} }, "size": 0 }))+ .unwrap(),+ )+ .await+ .expect("search");+ assert_eq!(+ all.total.value, count as u64,+ "every document must be searchable after reopen"+ );+ }++ {+ let engine = Engine::new(config.clone()).expect("engine");+ engine+ .create_index_with_settings("pinned", Schema::empty(), settings_with(json!(1)))+ .expect("create pinned index");+ engine+ .create_index("spread", Schema::empty())+ .expect("create default index");++ let idx = engine.get_index("pinned").expect("pinned index");+ write_docs(&idx, 0..25).await;+ write_docs(&engine.get_index("spread").expect("spread index"), 0..1).await;++ assert_eq!(+ wal_shard_dirs(&pinned_dir),+ 0,+ "one WAL shard means no s{{N}} subdirectories"+ );+ assert_eq!(+ root_wal_files(&pinned_dir),+ 1,+ "one WAL shard means exactly one WAL file"+ );+ // Control: without the setting the same engine config opens the+ // core-scaled layout, so the assertions above are driven by the+ // setting and not by an engine that only ever opens one shard.+ assert_eq!(wal_shard_dirs(&default_dir), 8);+ }++ {+ let engine = Engine::new(config.clone()).expect("reopen engine");+ let idx = engine+ .get_index("pinned")+ .expect("pinned index after reopen");+ assert_docs(&idx, 25).await;++ assert_eq!(+ wal_shard_dirs(&pinned_dir),+ 0,+ "reopen must honor the persisted shard count, not engine.ingest_shards"+ );+ assert_eq!(+ wal_shard_dirs(&default_dir),+ 8,+ "an index without the setting keeps the default"+ );++ // Writes taken by the *reopened* store must land in the same layout —+ // this is where a create/open mismatch would split the WAL.+ write_docs(&idx, 25..50).await;+ assert_eq!(wal_shard_dirs(&pinned_dir), 0);+ assert_eq!(root_wal_files(&pinned_dir), 1);+ }++ {+ let engine = Engine::new(config).expect("second reopen engine");+ let idx = engine+ .get_index("pinned")+ .expect("pinned index after second reopen");+ assert_docs(&idx, 50).await;+ assert_eq!(wal_shard_dirs(&pinned_dir), 0);+ }+}