Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

test(wal-shards): gate the per-index WAL shard override that 5f26032 shipped untested

xerj-org0fd3287ca626
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index b9f95b939..bfa2519e3 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -21458,7 +21458,8 @@ fn parse_passage_guard(ids: &Value) -> Option<HashSet<String>> {
/// file descriptor instead of one per ingest shard — the ingest-shard count
/// scales with CPU cores, so N indices held ~N×cores WAL fds open and exhausted
/// the (low, ~256) macOS file-descriptor limit at a few hundred datasets.
-/// Absent => the global `engine.ingest_shards` default (backward compatible).
+/// Absent, or out of `1..=MAX_PER_INDEX_WAL_SHARDS` => the global
+/// `engine.ingest_shards` default (backward compatible).
///
/// MUST be persisted at create and honored on open so the WAL *write* layout
/// stays consistent (doc routing is `xxh3(id) & (num_shards-1)`). Replay itself
@@ -21469,10 +21470,19 @@ fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {
settings
.pointer("/index/xerj_ingest_shards")
.and_then(Value::as_u64)
- .filter(|&n| n >= 1)
+ .filter(|&n| (1..=MAX_PER_INDEX_WAL_SHARDS).contains(&n))
.map(|n| n as usize)
}
+/// Ceiling for the per-index override, mirroring the `engine.ingest_shards`
+/// limit enforced by `EngineConfig::validate`. The global setting is validated
+/// at startup; this one arrives in an index-create request body (es_compat
+/// threads `settings` through untouched) and `IndexStore::open` eagerly creates
+/// a directory and opens a WAL file descriptor per shard, so an unbounded value
+/// is a create-time fd exhaustion — the exact failure this setting exists to
+/// prevent.
+const MAX_PER_INDEX_WAL_SHARDS: u64 = 256;
+
fn store_config_from(config: &Config, wal_shards_override: Option<usize>) -> IndexStoreConfig {
let sync_mode = match config.storage.wal_sync {
xerj_common::config::WalSync::Sync => SyncMode::Strict,
@@ -32512,3 +32522,9 @@ mod flush_memory_integration_tests {
// its own file (src/graph.rs) instead of growing this one.
#[path = "graph.rs"]
pub mod graph;
+
+// Child module (same `#[path]` reason as `graph`): the WAL-shard override
+// helpers are private free functions of this module.
+#[cfg(test)]
+#[path = "wal_shard_settings_tests.rs"]
+mod wal_shard_settings_tests;
diff --git a/engine/crates/xerj-engine/src/wal_shard_settings_tests.rs b/engine/crates/xerj-engine/src/wal_shard_settings_tests.rs
new file mode 100644
index 000000000..e2b74c97a
--- /dev/null
+++ b/engine/crates/xerj-engine/src/wal_shard_settings_tests.rs
@@ -0,0 +1,262 @@
+//! Regressions for the per-index WAL shard override (`index.xerj_ingest_shards`).
+//!
+//! The override exists because each index eagerly opens one WAL file descriptor
+//! per ingest shard, and the ingest-shard count scales with CPU cores: a few
+//! hundred autoindex datasets exhausted the macOS fd limit. Two properties have
+//! to hold or the fix is worse than the bug — the value must be *ignored* unless
+//! it is a sane count, and it must be honored identically at create and on
+//! reopen, since the WAL write layout (`wal/*.wal` vs `wal/s{N}/*.wal`) and doc
+//! routing both derive from it.
+
+use std::path::Path;
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_common::types::Schema;
+
+use super::{store_config_from, wal_shards_override_from_settings};
+use crate::Engine;
+
+fn settings_with(shards: Value) -> Value {
+ json!({ "index": { "xerj_ingest_shards": shards } })
+}
+
+/// The `s{N}` shard subdirectories under an index's WAL root. Zero of them means
+/// the single-shard (legacy root) layout; `IndexStore::open` creates one per
+/// shard whenever the count is > 1, so this is the on-disk witness of the shard
+/// count the store was actually opened with.
+fn wal_shard_dirs(index_dir: &Path) -> usize {
+ std::fs::read_dir(index_dir.join("wal"))
+ .expect("wal dir")
+ .flatten()
+ .filter(|e| {
+ e.path().is_dir()
+ && e.file_name()
+ .to_string_lossy()
+ .strip_prefix('s')
+ .is_some_and(|n| n.parse::<usize>().is_ok())
+ })
+ .count()
+}
+
+fn root_wal_files(index_dir: &Path) -> usize {
+ std::fs::read_dir(index_dir.join("wal"))
+ .expect("wal dir")
+ .flatten()
+ .filter(|e| e.path().extension().is_some_and(|ext| ext == "wal"))
+ .count()
+}
+
+/// `engine.ingest_shards` defaults to a fraction of the host's core count, so a
+/// test that let it default would silently assert nothing on a 1- or 2-core
+/// runner (the default would already be 1). Pin it well above 1.
+fn config_for(dir: &TempDir) -> Config {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.engine.ingest_shards = 8;
+ config
+}
+
+#[test]
+fn a_positive_wal_shard_count_in_index_settings_is_used() {
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(json!(1))),
+ Some(1)
+ );
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(json!(4))),
+ Some(4)
+ );
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(json!(256))),
+ Some(256),
+ "256 is the documented ceiling and must still be accepted"
+ );
+}
+
+#[test]
+fn an_absent_wal_shard_setting_leaves_the_engine_default_in_charge() {
+ for absent in [
+ Value::Null,
+ json!({}),
+ json!({ "index": {} }),
+ json!({ "index": { "number_of_shards": 1 } }),
+ json!({ "xerj_ingest_shards": 1 }),
+ json!({ "index.xerj_ingest_shards": 1 }),
+ ] {
+ assert_eq!(
+ wal_shards_override_from_settings(&absent),
+ None,
+ "no override should be read from {absent}"
+ );
+ }
+}
+
+#[test]
+fn zero_and_negative_wal_shard_counts_are_ignored_never_used_as_a_count() {
+ for bad in [json!(0), json!(-1), json!(-8)] {
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(bad.clone())),
+ None,
+ "{bad} must not become a shard count"
+ );
+ }
+}
+
+#[test]
+fn non_numeric_wal_shard_values_are_ignored_never_coerced() {
+ for bad in [
+ json!("4"),
+ json!("many"),
+ json!(4.5),
+ json!(true),
+ json!([4]),
+ json!({ "value": 4 }),
+ Value::Null,
+ ] {
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(bad.clone())),
+ None,
+ "{bad} must not become a shard count"
+ );
+ }
+}
+
+/// The value arrives straight off an index-create request body, and every shard
+/// costs a directory plus a permanently open WAL file descriptor at
+/// `IndexStore::open`. An out-of-range count is refused outright rather than
+/// clamped, so a typo falls back to the engine default instead of quietly
+/// running with a shard count nobody asked for.
+#[test]
+fn absurdly_large_wal_shard_counts_are_refused_and_never_opened() {
+ for absurd in [json!(257), json!(100_000), json!(u64::MAX)] {
+ assert_eq!(
+ wal_shards_override_from_settings(&settings_with(absurd.clone())),
+ None,
+ "{absurd} exceeds the 256 ceiling and must be refused"
+ );
+ }
+}
+
+#[test]
+fn the_store_config_takes_its_wal_shard_count_from_the_override_when_present() {
+ let dir = TempDir::new().expect("temp dir");
+ let config = config_for(&dir);
+
+ assert_eq!(store_config_from(&config, None).num_wal_shards, 8);
+ assert_eq!(store_config_from(&config, Some(1)).num_wal_shards, 1);
+ assert_eq!(store_config_from(&config, Some(2)).num_wal_shards, 2);
+}
+
+/// The whole point of persisting the setting: create and reopen must agree on
+/// the shard count. If reopen fell back to `engine.ingest_shards`, the reopened
+/// store would write into `wal/s{N}/` while the create-time entries live in
+/// `wal/`, splitting the layout under a single index.
+#[tokio::test]
+async fn a_pinned_wal_shard_count_survives_reopen_with_every_document_intact() {
+ let dir = TempDir::new().expect("temp dir");
+ let config = config_for(&dir);
+ let pinned_dir = dir.path().join("pinned");
+ let default_dir = dir.path().join("spread");
+
+ // Deliberately below the flush thresholds: the documents stay in the
+ // memtable + WAL, so surviving a reopen means the WAL was replayed out of
+ // the layout the persisted shard count selected, not read back from a
+ // segment that would have survived either way.
+ async fn write_docs(idx: &crate::Index, range: std::ops::Range<u32>) {
+ for i in range {
+ idx.index_document(
+ Some(format!("doc-{i}")),
+ json!({ "body": format!("payload {i}") }),
+ )
+ .await
+ .expect("index document");
+ }
+ }
+
+ async fn assert_docs(idx: &crate::Index, count: u32) {
+ for i in 0..count {
+ let doc = idx
+ .get_document(&format!("doc-{i}"))
+ .await
+ .expect("get document")
+ .unwrap_or_else(|| panic!("doc-{i} was lost"));
+ assert_eq!(doc["body"], json!(format!("payload {i}")));
+ }
+ let all = idx
+ .search(
+ &xerj_query::parse_request(&json!({ "query": { "match_all": {} }, "size": 0 }))
+ .unwrap(),
+ )
+ .await
+ .expect("search");
+ assert_eq!(
+ all.total.value, count as u64,
+ "every document must be searchable after reopen"
+ );
+ }
+
+ {
+ let engine = Engine::new(config.clone()).expect("engine");
+ engine
+ .create_index_with_settings("pinned", Schema::empty(), settings_with(json!(1)))
+ .expect("create pinned index");
+ engine
+ .create_index("spread", Schema::empty())
+ .expect("create default index");
+
+ let idx = engine.get_index("pinned").expect("pinned index");
+ write_docs(&idx, 0..25).await;
+ write_docs(&engine.get_index("spread").expect("spread index"), 0..1).await;
+
+ assert_eq!(
+ wal_shard_dirs(&pinned_dir),
+ 0,
+ "one WAL shard means no s{{N}} subdirectories"
+ );
+ assert_eq!(
+ root_wal_files(&pinned_dir),
+ 1,
+ "one WAL shard means exactly one WAL file"
+ );
+ // Control: without the setting the same engine config opens the
+ // core-scaled layout, so the assertions above are driven by the
+ // setting and not by an engine that only ever opens one shard.
+ assert_eq!(wal_shard_dirs(&default_dir), 8);
+ }
+
+ {
+ let engine = Engine::new(config.clone()).expect("reopen engine");
+ let idx = engine
+ .get_index("pinned")
+ .expect("pinned index after reopen");
+ assert_docs(&idx, 25).await;
+
+ assert_eq!(
+ wal_shard_dirs(&pinned_dir),
+ 0,
+ "reopen must honor the persisted shard count, not engine.ingest_shards"
+ );
+ assert_eq!(
+ wal_shard_dirs(&default_dir),
+ 8,
+ "an index without the setting keeps the default"
+ );
+
+ // Writes taken by the *reopened* store must land in the same layout —
+ // this is where a create/open mismatch would split the WAL.
+ write_docs(&idx, 25..50).await;
+ assert_eq!(wal_shard_dirs(&pinned_dir), 0);
+ assert_eq!(root_wal_files(&pinned_dir), 1);
+ }
+
+ {
+ let engine = Engine::new(config).expect("second reopen engine");
+ let idx = engine
+ .get_index("pinned")
+ .expect("pinned index after second reopen");
+ assert_docs(&idx, 50).await;
+ assert_eq!(wal_shard_dirs(&pinned_dir), 0);
+ }
+}