AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): bound per-index WAL fds so large repos don't exhaust the macOS FD limit
xerj-org5f260325dd67
diff --git a/.github/scripts/autoindex-fd-smoke.sh b/.github/scripts/autoindex-fd-smoke.shnew file mode 100755index 000000000..b1bf7bc30--- /dev/null+++ b/.github/scripts/autoindex-fd-smoke.sh@@ -0,0 +1,82 @@+#!/usr/bin/env bash+# Per-platform regression guard for the autoindex "Too many open files" crash.+#+# The crash: `xerj autoindex` on a large repo infers hundreds of datasets =>+# hundreds of indices. Before the fix each index pinned one WAL file descriptor+# PER ingest shard (a count that scales with CPU cores, ~8-16), so a few hundred+# datasets held thousands of fds open and aborted with EMFILE — fatal on macOS,+# whose default soft limit is 256.+#+# This gate constrains the descriptor budget so the regression is caught on ANY+# runner regardless of its `kern.maxfilesperproc`:+# soft 256 — the macOS default; the server MUST raise it (raise_nofile_limit)+# hard 4096 — between the pre-fix need (~6,400 for 400 datasets) and the+# post-fix need (~600). Pre-fix => EMFILE; post-fix => fits.+# So the job fails if EITHER the limit-raise OR the per-index WAL-fd reduction+# regresses. Runs under bash on ubuntu / macOS / windows(git-bash).+set -uo pipefail++# Best-effort FD budget (no-op on Windows, which uses a different handle model).+ulimit -Hn 4096 2>/dev/null || true+ulimit -Sn 256 2>/dev/null || true+echo "platform: $(uname -s) ulimit -Sn: $(ulimit -Sn 2>/dev/null || echo n/a) -Hn: $(ulimit -Hn 2>/dev/null || echo n/a)"++BIN="engine/target/release/xerj"+[ -f "$BIN.exe" ] && BIN="$BIN.exe" # windows++# ── generate 400 distinct-schema datasets (one index each) ─────────────────+ROOT="$(mktemp -d)"+CORPUS="$ROOT/corpus"+mkdir -p "$CORPUS"+for i in $(seq 1 400); do+ d="$CORPUS/ds_$i"+ mkdir -p "$d"+ # Unique column names per folder => a distinct inferred dataset per folder.+ printf 'k_%d_id,k_%d_name,k_%d_val\n1,alpha,10\n2,beta,20\n3,gamma,30\n' "$i" "$i" "$i" > "$d/data.csv"+done+echo "generated $(find "$CORPUS" -name '*.csv' | wc -l | tr -d ' ') dataset files"++# ── boot the server (inherits the constrained fd budget) ───────────────────+DATA="$ROOT/data"+mkdir -p "$DATA"+"$BIN" --insecure --data-dir "$DATA" > "$DATA/server.log" 2>&1 &+SPID=$!+up=0+for _ in $(seq 1 160); do+ if curl -fs -m1 localhost:9200/_cluster/health >/dev/null 2>&1; then up=1; break; fi+ kill -0 "$SPID" 2>/dev/null || { echo "server exited during boot"; cat "$DATA/server.log"; exit 1; }+ sleep 0.5+done+[ "$up" = 1 ] || { echo "server never became healthy"; cat "$DATA/server.log"; exit 1; }++# ── autoindex the corpus ───────────────────────────────────────────────────+"$BIN" autoindex "$CORPUS" > "$DATA/ax.log" 2>&1+RC=$?+echo "autoindex exit=$RC"+tail -20 "$DATA/ax.log" || true++# ── assertions ─────────────────────────────────────────────────────────────+FAIL=0+if grep -qiE 'too many open files|os error 24|emfile' "$DATA/ax.log" "$DATA/server.log"; then+ echo "::error::FD-exhaustion regression — autoindex hit 'Too many open files'"+ FAIL=1+fi+# autoindex exit codes: 0 complete, 3 completed-with-junk (both fine).+if [ "$RC" != "0" ] && [ "$RC" != "3" ]; then+ echo "::error::autoindex exited $RC (expected 0 or 3)"+ FAIL=1+fi+CREATED="$(curl -s 'localhost:9200/_cat/indices?h=index' 2>/dev/null | grep -c 'ax-' || true)"+echo "ax-* indices created: $CREATED"+if [ "${CREATED:-0}" -lt 300 ]; then+ echo "::error::expected >=300 datasets indexed, got ${CREATED:-0}"+ FAIL=1+fi++kill "$SPID" 2>/dev/null || true+if [ "$FAIL" = 0 ]; then+ echo "AUTOINDEX FD SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE)"+else+ echo "AUTOINDEX FD SMOKE FAILED on $(uname -s)"+ exit 1+fidiff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 1daae4af7..50396932d 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -197,3 +197,41 @@ jobs:- name: Stop xerjif: always()run: kill "${XERJ_PID:-}" 2>/dev/null || true++ # Per-OS runtime gate. The earlier autoindex "Too many open files" crash was+ # macOS-only and slipped through because nothing in CI actually RAN the binary+ # on macOS/Windows. This job boots xerj and autoindexes a many-dataset corpus+ # on each OS under a constrained descriptor budget, failing on any EMFILE.+ # macOS runners default to `ulimit -n 256` — the exact condition it regressed+ # under.+ autoindex-fd-smoke:+ name: Autoindex FD smoke (${{ matrix.os }})+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, macos-latest, windows-latest]+ runs-on: ${{ matrix.os }}+ steps:+ - uses: actions/checkout@v4++ - name: Install Rust+ uses: dtolnay/rust-toolchain@stable++ - name: Cache+ uses: actions/cache@v4+ with:+ path: |+ ~/.cargo/registry/cache+ ~/.cargo/registry/index+ ~/.cargo/git/db+ engine/target+ key: ci-smoke-${{ matrix.os }}-${{ hashFiles('engine/Cargo.lock') }}+ restore-keys: ci-smoke-${{ matrix.os }}-++ - name: Build xerj (release)+ working-directory: engine+ run: cargo build --release -p xerj-server++ - name: Autoindex a many-dataset corpus (must not exhaust file descriptors)+ shell: bash+ run: bash .github/scripts/autoindex-fd-smoke.shdiff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex e205beef9..3d1199512 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -744,7 +744,16 @@ pub async fn create_index(}).unwrap_or_else(|| "asc".to_string());- match state.engine.create_index(&index, schema) {+ // Thread the request's `settings` into the store config at create time+ // (custom analysis config, and the per-index `index.xerj_ingest_shards` WAL+ // shard override that `xerj autoindex` uses to keep each of its hundreds of+ // indices at a single WAL file descriptor). create_index_with_settings+ // applies index templates identically to create_index.+ let create_settings = body.get("settings").cloned().unwrap_or(Value::Null);+ match state+ .engine+ .create_index_with_settings(&index, schema, create_settings)+ {Ok(()) => {// Store the raw settings / mappings / aliases blob as written.// These are used by `GET /{index}/_settings`, `GET /{index}/_mapping`,diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex 41795af5d..a71d83f18 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -24,6 +24,31 @@ pub struct BulkOutcome {pub first_server_error: Option<String>,}+/// Ensure an autoindex index-create body pins the index to a single WAL shard.+///+/// autoindex creates one index per inferred dataset — hundreds for a large+/// repo. Each index otherwise opens one WAL file per ingest shard (a count that+/// scales with the server's CPU cores), so hundreds of indices exhaust the+/// process file-descriptor limit — fatal on macOS, whose default soft limit is+/// 256. `index.xerj_ingest_shards = 1` keeps each index at a single WAL fd.+/// Non-destructive: only fills the key when absent.+fn with_single_wal_shard(body: &Value) -> Value {+ let mut body = body.clone();+ if let Some(obj) = body.as_object_mut() {+ let settings = obj+ .entry("settings")+ .or_insert_with(|| serde_json::json!({}));+ if let Some(sobj) = settings.as_object_mut() {+ let index = sobj.entry("index").or_insert_with(|| serde_json::json!({}));+ if let Some(iobj) = index.as_object_mut() {+ iobj.entry("xerj_ingest_shards")+ .or_insert_with(|| serde_json::json!(1));+ }+ }+ }+ body+}+impl Es {pub fn new(url: &str, api_key: Option<String>) -> Result<Self> {Self::with_bulk_timeout(url, api_key, 300)@@ -128,9 +153,10 @@ impl Es {/// PUT index with explicit mapping; tolerates already-exists.pub fn ensure_index(&self, index: &str, body: &Value) -> Result<()> {+ let body = with_single_wal_shard(body);let resp = self.req(reqwest::Method::PUT, &format!("/{index}"))- .json(body)+ .json(&body).send().context("PUT index")?;let status = resp.status();diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 76cd7084b..98b2beec7 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -714,20 +714,17 @@ impl Engine {/// Create a new index, applying any matching index template.pub fn create_index(&self, name: &str, schema: Schema) -> Result<()> {- let index_name = IndexName::new(name).map_err(EngineError::Common)?;-- if self.indices.contains_key(name) {- return Err(EngineError::Common(- xerj_common::XerjError::index_already_exists(name),- ));- }+ self.create_index_with_settings(name, schema, Value::Null)+ }- // Apply matching template (highest priority wins).+ /// Merge the highest-priority matching index template's declared fields+ /// into `schema` (adding any the schema does not already define). Applied+ /// on every create path so template behavior is independent of whether the+ /// caller supplied settings.+ fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {let mut effective_schema = schema;if let Some(tmpl) = self.best_matching_template(name) {- // Merge template mappings into schema.if let Some(props) = tmpl.mappings.get("properties") {- // Parse template properties and add any missing fields.if let Some(obj) = props.as_object() {for (field_name, field_def) in obj {let es_type = field_def@@ -750,11 +747,7 @@ impl Engine {}}}-- let idx = Index::create(index_name, effective_schema, &self.config, &self.data_dir)?;- self.indices.insert(name.to_string(), idx);- info!(name, "index created");- Ok(())+ effective_schema}/// Create a new index with explicit settings (e.g. custom analysis configuration).@@ -775,9 +768,11 @@ impl Engine {));}+ // Apply matching template (highest priority wins) on every create path.+ let effective_schema = self.apply_index_template(name, schema);let idx = Index::create_with_settings(index_name,- schema,+ effective_schema,settings,&self.config,&self.data_dir,diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex a3e8d78cd..9a01831f8 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -3283,7 +3283,7 @@ impl Index {);std::fs::create_dir_all(&index_dir)?;- let store_config = store_config_from(config);+ let store_config = store_config_from(config, wal_shards_override_from_settings(&settings));let store = IndexStore::open(&index_dir, store_config)?;let managed = ManagedSchema {@@ -3478,7 +3478,11 @@ impl Index {));}- let store_config = store_config_from(config);+ // Load persisted settings BEFORE opening the store so the WAL shard+ // count (index.xerj_ingest_shards) matches what create used — the WAL+ // write layout (root vs s{N}/) and doc routing depend on it.+ let settings = load_settings(&index_dir).unwrap_or(Value::Null);+ let store_config = store_config_from(config, wal_shards_override_from_settings(&settings));let store = IndexStore::open(&index_dir, store_config)?;// Estimate doc count from snapshot.@@ -3497,8 +3501,8 @@ impl Index {)?;let effective_embedder = make_embedder_for_schema(&schema.schema, &config.embedding)?;- // Load persisted settings early so we can build the registry before WAL replay.- let settings = load_settings(&index_dir).unwrap_or(Value::Null);+ // `settings` was loaded above (before the store was opened) so the WAL+ // shard count, the analyzer registry, and WAL replay all agree.// Build analyzer registry from persisted settings so WAL replay uses// the same custom analyzers (synonyms, ngrams, etc.) that were active@@ -21384,7 +21388,28 @@ fn parse_passage_guard(ids: &Value) -> Option<HashSet<String>> {}}-fn store_config_from(config: &Config) -> IndexStoreConfig {+/// Per-index WAL shard count read from index settings+/// (`settings.index.xerj_ingest_shards`), if set. `xerj autoindex` sets this to+/// 1 so each of its (potentially hundreds of) small indices pins a single WAL+/// file descriptor instead of one per ingest shard — the ingest-shard count+/// scales with CPU cores, so N indices held ~N×cores WAL fds open and exhausted+/// the (low, ~256) macOS file-descriptor limit at a few hundred datasets.+/// Absent => the global `engine.ingest_shards` default (backward compatible).+///+/// MUST be persisted at create and honored on open so the WAL *write* layout+/// stays consistent (doc routing is `xxh3(id) & (num_shards-1)`). Replay itself+/// is self-describing (`wal::replay_all_sorted` discovers both the root and+/// `s{N}/` layouts from disk), so a mismatch cannot lose data, but honoring the+/// persisted value keeps writes from splitting across two layouts.+fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {+ settings+ .pointer("/index/xerj_ingest_shards")+ .and_then(Value::as_u64)+ .filter(|&n| n >= 1)+ .map(|n| n as usize)+}++fn store_config_from(config: &Config, wal_shards_override: Option<usize>) -> IndexStoreConfig {let sync_mode = match config.storage.wal_sync {xerj_common::config::WalSync::Sync => SyncMode::Strict,xerj_common::config::WalSync::Batched | xerj_common::config::WalSync::Async => {@@ -21406,7 +21431,7 @@ fn store_config_from(config: &Config) -> IndexStoreConfig {wal_batch_ms,schema_version: 1,storage_mode: xerj_storage::StorageMode::Local,- num_wal_shards: config.engine.ingest_shards,+ num_wal_shards: wal_shards_override.unwrap_or(config.engine.ingest_shards),}}