Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): bound per-index WAL fds so large repos don't exhaust the macOS FD limit

xerj-org5f260325dd67
diff --git a/.github/scripts/autoindex-fd-smoke.sh b/.github/scripts/autoindex-fd-smoke.sh
new file mode 100755
index 000000000..b1bf7bc30
--- /dev/null
+++ b/.github/scripts/autoindex-fd-smoke.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Per-platform regression guard for the autoindex "Too many open files" crash.
+#
+# The crash: `xerj autoindex` on a large repo infers hundreds of datasets =>
+# hundreds of indices. Before the fix each index pinned one WAL file descriptor
+# PER ingest shard (a count that scales with CPU cores, ~8-16), so a few hundred
+# datasets held thousands of fds open and aborted with EMFILE — fatal on macOS,
+# whose default soft limit is 256.
+#
+# This gate constrains the descriptor budget so the regression is caught on ANY
+# runner regardless of its `kern.maxfilesperproc`:
+# soft 256 — the macOS default; the server MUST raise it (raise_nofile_limit)
+# hard 4096 — between the pre-fix need (~6,400 for 400 datasets) and the
+# post-fix need (~600). Pre-fix => EMFILE; post-fix => fits.
+# So the job fails if EITHER the limit-raise OR the per-index WAL-fd reduction
+# regresses. Runs under bash on ubuntu / macOS / windows(git-bash).
+set -uo pipefail
+
+# Best-effort FD budget (no-op on Windows, which uses a different handle model).
+ulimit -Hn 4096 2>/dev/null || true
+ulimit -Sn 256 2>/dev/null || true
+echo "platform: $(uname -s) ulimit -Sn: $(ulimit -Sn 2>/dev/null || echo n/a) -Hn: $(ulimit -Hn 2>/dev/null || echo n/a)"
+
+BIN="engine/target/release/xerj"
+[ -f "$BIN.exe" ] && BIN="$BIN.exe" # windows
+
+# ── generate 400 distinct-schema datasets (one index each) ─────────────────
+ROOT="$(mktemp -d)"
+CORPUS="$ROOT/corpus"
+mkdir -p "$CORPUS"
+for i in $(seq 1 400); do
+ d="$CORPUS/ds_$i"
+ mkdir -p "$d"
+ # Unique column names per folder => a distinct inferred dataset per folder.
+ printf 'k_%d_id,k_%d_name,k_%d_val\n1,alpha,10\n2,beta,20\n3,gamma,30\n' "$i" "$i" "$i" > "$d/data.csv"
+done
+echo "generated $(find "$CORPUS" -name '*.csv' | wc -l | tr -d ' ') dataset files"
+
+# ── boot the server (inherits the constrained fd budget) ───────────────────
+DATA="$ROOT/data"
+mkdir -p "$DATA"
+"$BIN" --insecure --data-dir "$DATA" > "$DATA/server.log" 2>&1 &
+SPID=$!
+up=0
+for _ in $(seq 1 160); do
+ if curl -fs -m1 localhost:9200/_cluster/health >/dev/null 2>&1; then up=1; break; fi
+ kill -0 "$SPID" 2>/dev/null || { echo "server exited during boot"; cat "$DATA/server.log"; exit 1; }
+ sleep 0.5
+done
+[ "$up" = 1 ] || { echo "server never became healthy"; cat "$DATA/server.log"; exit 1; }
+
+# ── autoindex the corpus ───────────────────────────────────────────────────
+"$BIN" autoindex "$CORPUS" > "$DATA/ax.log" 2>&1
+RC=$?
+echo "autoindex exit=$RC"
+tail -20 "$DATA/ax.log" || true
+
+# ── assertions ─────────────────────────────────────────────────────────────
+FAIL=0
+if grep -qiE 'too many open files|os error 24|emfile' "$DATA/ax.log" "$DATA/server.log"; then
+ echo "::error::FD-exhaustion regression — autoindex hit 'Too many open files'"
+ FAIL=1
+fi
+# autoindex exit codes: 0 complete, 3 completed-with-junk (both fine).
+if [ "$RC" != "0" ] && [ "$RC" != "3" ]; then
+ echo "::error::autoindex exited $RC (expected 0 or 3)"
+ FAIL=1
+fi
+CREATED="$(curl -s 'localhost:9200/_cat/indices?h=index' 2>/dev/null | grep -c 'ax-' || true)"
+echo "ax-* indices created: $CREATED"
+if [ "${CREATED:-0}" -lt 300 ]; then
+ echo "::error::expected >=300 datasets indexed, got ${CREATED:-0}"
+ FAIL=1
+fi
+
+kill "$SPID" 2>/dev/null || true
+if [ "$FAIL" = 0 ]; then
+ echo "AUTOINDEX FD SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE)"
+else
+ echo "AUTOINDEX FD SMOKE FAILED on $(uname -s)"
+ exit 1
+fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1daae4af7..50396932d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -197,3 +197,41 @@ jobs:
- name: Stop xerj
if: always()
run: kill "${XERJ_PID:-}" 2>/dev/null || true
+
+ # Per-OS runtime gate. The earlier autoindex "Too many open files" crash was
+ # macOS-only and slipped through because nothing in CI actually RAN the binary
+ # on macOS/Windows. This job boots xerj and autoindexes a many-dataset corpus
+ # on each OS under a constrained descriptor budget, failing on any EMFILE.
+ # macOS runners default to `ulimit -n 256` — the exact condition it regressed
+ # under.
+ autoindex-fd-smoke:
+ name: Autoindex FD smoke (${{ matrix.os }})
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, macos-latest, windows-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Install Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.cargo/registry/cache
+ ~/.cargo/registry/index
+ ~/.cargo/git/db
+ engine/target
+ key: ci-smoke-${{ matrix.os }}-${{ hashFiles('engine/Cargo.lock') }}
+ restore-keys: ci-smoke-${{ matrix.os }}-
+
+ - name: Build xerj (release)
+ working-directory: engine
+ run: cargo build --release -p xerj-server
+
+ - name: Autoindex a many-dataset corpus (must not exhaust file descriptors)
+ shell: bash
+ run: bash .github/scripts/autoindex-fd-smoke.sh
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index e205beef9..3d1199512 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -744,7 +744,16 @@ pub async fn create_index(
})
.unwrap_or_else(|| "asc".to_string());
- match state.engine.create_index(&index, schema) {
+ // Thread the request's `settings` into the store config at create time
+ // (custom analysis config, and the per-index `index.xerj_ingest_shards` WAL
+ // shard override that `xerj autoindex` uses to keep each of its hundreds of
+ // indices at a single WAL file descriptor). create_index_with_settings
+ // applies index templates identically to create_index.
+ let create_settings = body.get("settings").cloned().unwrap_or(Value::Null);
+ match state
+ .engine
+ .create_index_with_settings(&index, schema, create_settings)
+ {
Ok(()) => {
// Store the raw settings / mappings / aliases blob as written.
// These are used by `GET /{index}/_settings`, `GET /{index}/_mapping`,
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index 41795af5d..a71d83f18 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -24,6 +24,31 @@ pub struct BulkOutcome {
pub first_server_error: Option<String>,
}
+/// Ensure an autoindex index-create body pins the index to a single WAL shard.
+///
+/// autoindex creates one index per inferred dataset — hundreds for a large
+/// repo. Each index otherwise opens one WAL file per ingest shard (a count that
+/// scales with the server's CPU cores), so hundreds of indices exhaust the
+/// process file-descriptor limit — fatal on macOS, whose default soft limit is
+/// 256. `index.xerj_ingest_shards = 1` keeps each index at a single WAL fd.
+/// Non-destructive: only fills the key when absent.
+fn with_single_wal_shard(body: &Value) -> Value {
+ let mut body = body.clone();
+ if let Some(obj) = body.as_object_mut() {
+ let settings = obj
+ .entry("settings")
+ .or_insert_with(|| serde_json::json!({}));
+ if let Some(sobj) = settings.as_object_mut() {
+ let index = sobj.entry("index").or_insert_with(|| serde_json::json!({}));
+ if let Some(iobj) = index.as_object_mut() {
+ iobj.entry("xerj_ingest_shards")
+ .or_insert_with(|| serde_json::json!(1));
+ }
+ }
+ }
+ body
+}
+
impl Es {
pub fn new(url: &str, api_key: Option<String>) -> Result<Self> {
Self::with_bulk_timeout(url, api_key, 300)
@@ -128,9 +153,10 @@ impl Es {
/// PUT index with explicit mapping; tolerates already-exists.
pub fn ensure_index(&self, index: &str, body: &Value) -> Result<()> {
+ let body = with_single_wal_shard(body);
let resp = self
.req(reqwest::Method::PUT, &format!("/{index}"))
- .json(body)
+ .json(&body)
.send()
.context("PUT index")?;
let status = resp.status();
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 76cd7084b..98b2beec7 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -714,20 +714,17 @@ impl Engine {
/// Create a new index, applying any matching index template.
pub fn create_index(&self, name: &str, schema: Schema) -> Result<()> {
- let index_name = IndexName::new(name).map_err(EngineError::Common)?;
-
- if self.indices.contains_key(name) {
- return Err(EngineError::Common(
- xerj_common::XerjError::index_already_exists(name),
- ));
- }
+ self.create_index_with_settings(name, schema, Value::Null)
+ }
- // Apply matching template (highest priority wins).
+ /// Merge the highest-priority matching index template's declared fields
+ /// into `schema` (adding any the schema does not already define). Applied
+ /// on every create path so template behavior is independent of whether the
+ /// caller supplied settings.
+ fn apply_index_template(&self, name: &str, schema: Schema) -> Schema {
let mut effective_schema = schema;
if let Some(tmpl) = self.best_matching_template(name) {
- // Merge template mappings into schema.
if let Some(props) = tmpl.mappings.get("properties") {
- // Parse template properties and add any missing fields.
if let Some(obj) = props.as_object() {
for (field_name, field_def) in obj {
let es_type = field_def
@@ -750,11 +747,7 @@ impl Engine {
}
}
}
-
- let idx = Index::create(index_name, effective_schema, &self.config, &self.data_dir)?;
- self.indices.insert(name.to_string(), idx);
- info!(name, "index created");
- Ok(())
+ effective_schema
}
/// Create a new index with explicit settings (e.g. custom analysis configuration).
@@ -775,9 +768,11 @@ impl Engine {
));
}
+ // Apply matching template (highest priority wins) on every create path.
+ let effective_schema = self.apply_index_template(name, schema);
let idx = Index::create_with_settings(
index_name,
- schema,
+ effective_schema,
settings,
&self.config,
&self.data_dir,
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index a3e8d78cd..9a01831f8 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -3283,7 +3283,7 @@ impl Index {
);
std::fs::create_dir_all(&index_dir)?;
- let store_config = store_config_from(config);
+ let store_config = store_config_from(config, wal_shards_override_from_settings(&settings));
let store = IndexStore::open(&index_dir, store_config)?;
let managed = ManagedSchema {
@@ -3478,7 +3478,11 @@ impl Index {
));
}
- let store_config = store_config_from(config);
+ // Load persisted settings BEFORE opening the store so the WAL shard
+ // count (index.xerj_ingest_shards) matches what create used — the WAL
+ // write layout (root vs s{N}/) and doc routing depend on it.
+ let settings = load_settings(&index_dir).unwrap_or(Value::Null);
+ let store_config = store_config_from(config, wal_shards_override_from_settings(&settings));
let store = IndexStore::open(&index_dir, store_config)?;
// Estimate doc count from snapshot.
@@ -3497,8 +3501,8 @@ impl Index {
)?;
let effective_embedder = make_embedder_for_schema(&schema.schema, &config.embedding)?;
- // Load persisted settings early so we can build the registry before WAL replay.
- let settings = load_settings(&index_dir).unwrap_or(Value::Null);
+ // `settings` was loaded above (before the store was opened) so the WAL
+ // shard count, the analyzer registry, and WAL replay all agree.
// Build analyzer registry from persisted settings so WAL replay uses
// the same custom analyzers (synonyms, ngrams, etc.) that were active
@@ -21384,7 +21388,28 @@ fn parse_passage_guard(ids: &Value) -> Option<HashSet<String>> {
}
}
-fn store_config_from(config: &Config) -> IndexStoreConfig {
+/// Per-index WAL shard count read from index settings
+/// (`settings.index.xerj_ingest_shards`), if set. `xerj autoindex` sets this to
+/// 1 so each of its (potentially hundreds of) small indices pins a single WAL
+/// file descriptor instead of one per ingest shard — the ingest-shard count
+/// scales with CPU cores, so N indices held ~N×cores WAL fds open and exhausted
+/// the (low, ~256) macOS file-descriptor limit at a few hundred datasets.
+/// Absent => the global `engine.ingest_shards` default (backward compatible).
+///
+/// MUST be persisted at create and honored on open so the WAL *write* layout
+/// stays consistent (doc routing is `xxh3(id) & (num_shards-1)`). Replay itself
+/// is self-describing (`wal::replay_all_sorted` discovers both the root and
+/// `s{N}/` layouts from disk), so a mismatch cannot lose data, but honoring the
+/// persisted value keeps writes from splitting across two layouts.
+fn wal_shards_override_from_settings(settings: &Value) -> Option<usize> {
+ settings
+ .pointer("/index/xerj_ingest_shards")
+ .and_then(Value::as_u64)
+ .filter(|&n| n >= 1)
+ .map(|n| n as usize)
+}
+
+fn store_config_from(config: &Config, wal_shards_override: Option<usize>) -> IndexStoreConfig {
let sync_mode = match config.storage.wal_sync {
xerj_common::config::WalSync::Sync => SyncMode::Strict,
xerj_common::config::WalSync::Batched | xerj_common::config::WalSync::Async => {
@@ -21406,7 +21431,7 @@ fn store_config_from(config: &Config) -> IndexStoreConfig {
wal_batch_ms,
schema_version: 1,
storage_mode: xerj_storage::StorageMode::Local,
- num_wal_shards: config.engine.ingest_shards,
+ num_wal_shards: wal_shards_override.unwrap_or(config.engine.ingest_shards),
}
}