AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
security: Phase-2 hardening — snapshot escape, field-limit, magic-link TOCTOU, info disclosure, index-name boundary; perf: ONNX windowing
xerj-org1c4e01ddac1d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex e205beef9..77f820d27 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -675,6 +675,16 @@ pub async fn create_index(return ApiError::new(e).into_response();}+ // #80: make the path-traversal guarantee LOCAL at this handler boundary.+ // `Engine::create_index` also runs `IndexName::new`, but validating the name+ // here means the es_compat surface itself rejects `..`, path separators, NUL+ // and other malformed names up front — defense-in-depth, not a guarantee+ // that lives only in a distant call. Same accepted set as before (the engine+ // enforced the identical rule); the rejection is just local and obvious.+ if let Err(e) = xerj_common::IndexName::validate(&index) {+ return ApiError::new(e).into_response();+ }+// Validate mapping field types before creating — ES returns// mapper_parsing_exception for unsupported types.let mappings_val = body.get("mappings").cloned().unwrap_or(Value::Null);diff --git a/engine/crates/xerj-common/src/types.rs b/engine/crates/xerj-common/src/types.rsindex 905552061..985ded709 100644--- a/engine/crates/xerj-common/src/types.rs+++ b/engine/crates/xerj-common/src/types.rs@@ -93,7 +93,11 @@ impl IndexName {&self.0}- fn validate(name: &str) -> Result<(), XerjError> {+ /// Validate an index name against the traversal/charset rules without+ /// constructing an `IndexName`. Public so handler boundaries (e.g. the+ /// es_compat `create_index` surface, #80) can reject `..`/separators/NUL+ /// locally instead of relying solely on the deeper `IndexName::new` guard.+ pub fn validate(name: &str) -> Result<(), XerjError> {if name.len() > 255 {return Err(XerjError::invalid_mapping(format!("index name too long: {} chars (max 255)",diff --git a/engine/crates/xerj-console-api/src/auth/magic.rs b/engine/crates/xerj-console-api/src/auth/magic.rsindex ed6df65fe..cc024ce49 100644--- a/engine/crates/xerj-console-api/src/auth/magic.rs+++ b/engine/crates/xerj-console-api/src/auth/magic.rs@@ -207,6 +207,14 @@ pub async fn redeem(}let token_hash = sha256_hex(body.token.as_bytes());+ // #76 S5-3: serialize the single-use check and the mark-used commit. The+ // used-check (below) and `mark_magic_link_used` (get→delete→create) are+ // separated by several awaits with no exclusive lock, so two concurrent+ // redeems of one token could both pass the check and both mint a session.+ // Hold the gate for the rest of this function so check→consume is atomic.+ // `lock_owned` avoids tying the guard's lifetime to `state` across awaits.+ let _redeem_guard = state.redeem_gate.clone().lock_owned().await;+// Look it up.let link = store::get_magic_link(&state.engine, &token_hash).await?diff --git a/engine/crates/xerj-console-api/src/cluster.rs b/engine/crates/xerj-console-api/src/cluster.rsindex d89233d6a..deac26946 100644--- a/engine/crates/xerj-console-api/src/cluster.rs+++ b/engine/crates/xerj-console-api/src/cluster.rs@@ -28,12 +28,42 @@ pub async fn info(State(state): State<ConsoleState>) -> ConsoleResult<Response>ClusterMode::Raft => "raft",};- let body = json!({+ Ok(ok(info_body(mode, state.started_at.0), None))+}++/// Build the unauthenticated `cluster/info` body.+///+/// AUTHZ-2: this endpoint is unauthenticated (the SPA reads it at boot, before+/// login, to choose standalone-vs-topology view). It must therefore expose only+/// what that decision needs — `mode` — plus the benign uptime. `node_id` and+/// the exact build `version` are deliberately omitted: a pre-auth version string+/// hands an attacker a precise target for known-CVE matching, and the node+/// identity leaks cluster topology. Authenticated callers get the full detail+/// from the credentialed cluster/status paths.+fn info_body(mode: &str, started_at_ms: i64) -> serde_json::Value {+ json!({"mode": mode,- "node_id": state.node_id.as_str(),- "version": env!("CARGO_PKG_VERSION"),- "started_at_ms": state.started_at.0,- });+ "started_at_ms": started_at_ms,+ })+}++#[cfg(test)]+mod tests {+ use super::info_body;- Ok(ok(body, None))+ #[test]+ fn info_body_omits_node_id_and_version_pre_auth() {+ let v = info_body("standalone", 1_700_000_000_000);+ assert_eq!(v.get("mode").and_then(|m| m.as_str()), Some("standalone"));+ assert!(v.get("started_at_ms").is_some());+ // The AUTHZ-2 disclosure fields must be gone from the pre-auth body.+ assert!(v.get("node_id").is_none(), "node_id must not leak pre-auth");+ assert!(v.get("version").is_none(), "version must not leak pre-auth");+ // And the whole serialized body must not contain the build version.+ let s = v.to_string();+ assert!(+ !s.contains(env!("CARGO_PKG_VERSION")),+ "build version must not appear in the pre-auth body: {s}"+ );+ }}diff --git a/engine/crates/xerj-console-api/src/state.rs b/engine/crates/xerj-console-api/src/state.rsindex 1a836b6ed..932f1a3f3 100644--- a/engine/crates/xerj-console-api/src/state.rs+++ b/engine/crates/xerj-console-api/src/state.rs@@ -118,6 +118,15 @@ pub struct ConsoleState {/// `data_dir/.xerj_master_key` file or the `XERJ_CONSOLE_KEY`/// env var; generated on first start and persisted).pub master_key: Arc<[u8; 32]>,++ /// Serializes magic-link redemption (#76 S5-3). The single-use check and+ /// the `mark_magic_link_used` commit are separated by several awaits and+ /// the store's mark-used is itself get→delete→create, so two concurrent+ /// redeems of one token could both pass the used-check and both mint a+ /// session. Holding this gate across the whole check→consume makes the+ /// transition atomic. Redemptions are rare (enrollment/recovery), so a+ /// single gate is cheaper than a per-token map and needs no cleanup.+ pub redeem_gate: Arc<tokio::sync::Mutex<()>>,}#[derive(Debug, Clone, Copy)]@@ -159,6 +168,7 @@ impl ConsoleState {enrollment_sessions: Arc::new(DashMap::new()),auth_rate_counters: Arc::new(DashMap::new()),master_key: Arc::new(master_key),+ redeem_gate: Arc::new(tokio::sync::Mutex::new(())),}}}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 76cd7084b..36e788ef7 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1881,6 +1881,24 @@ fn validate_snapshot_path(// location itself is sane. Bound the location to `data_dir` (default) or a// configured allowlist (an ES `path.repo` equivalent), so a repo cannot point// at `/etc`, `/root/.ssh`, another tenant's dir, etc.+ // F-PATH-02 (residual): a location whose fully-resolved target does not yet+ // exist makes `canonicalize()` below fail, so the lexical fallback keeps any+ // `..` components. A component-based `starts_with` then treats+ // `<data_dir>/../../escape` as *inside* `<data_dir>` (it never normalizes+ // `..`), and `create_dir_all` lets the OS walk out. Reject `..` outright —+ // no legitimate snapshot root needs parent-dir traversal — so neither the+ // location containment check nor the snapshot-name check can be bypassed+ // with an unresolved parent reference.+ if std::path::Path::new(repo_path)+ .components()+ .any(|c| matches!(c, std::path::Component::ParentDir))+ {+ return Err(EngineError::Common(+ xerj_common::XerjError::invalid_mapping(format!(+ "snapshot repository location [{repo_path}] must not contain '..' path components"+ )),+ ));+ }let base_canon = |p: &std::path::Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf());let repo_base = base_canon(std::path::Path::new(repo_path));let mut allowed_bases: Vec<std::path::PathBuf> = vec![base_canon(data_dir)];@@ -2053,4 +2071,25 @@ mod snapshot_path_security_tests {let repo = data.path().to_str().unwrap();assert!(validate_snapshot_path(repo, "..", &snap, data.path(), &[]).is_err());}++ #[test]+ fn f_path_02_residual_parent_ref_to_nonexistent_target_is_rejected() {+ // The residual escape (#73): a location containing `..` whose fully+ // resolved target does NOT yet exist. canonicalize() fails, the lexical+ // fallback keeps the `..`, and a component-based starts_with(data_dir)+ // would pass — letting create_dir_all write outside data_dir. It must be+ // refused before any directory is created.+ let data = tempfile::TempDir::new().unwrap();+ let repo = format!("{}/../../xerj-escape", data.path().display());+ let snap = std::path::Path::new(&repo).join("s1");+ let err = validate_snapshot_path(&repo, "s1", &snap, data.path(), &[])+ .expect_err("a location containing `..` must be rejected");+ assert!(err.to_string().contains(".."), "unexpected error: {err}");+ // And it stays rejected even if the operator allowlists a broad root:+ // the `..` guard runs before containment, so a parent-ref cannot be+ // laundered through the allowlist either.+ let err2 = validate_snapshot_path(&repo, "s1", &snap, data.path(), &["/".to_string()])+ .expect_err("`..` must be rejected even with a permissive allowlist");+ assert!(err2.to_string().contains(".."), "unexpected error: {err2}");+ }}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex a3e8d78cd..fdc48e8f2 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -1251,6 +1251,44 @@ mod semantic_deadline_regression_tests {);}+ #[tokio::test]+ async fn explicit_add_fields_enforces_max_fields_per_index() {+ // #76 S5-5: the explicit mapping path (`add_fields` / PUT _mapping /+ // schema evolve) must enforce `max_fields_per_index` too. The dynamic+ // ingest path did, but `Index::add_fields` previously bypassed the cap,+ // so an authenticated caller could bloat the schema unbounded by sending+ // fields explicitly.+ let dir = TempDir::new().unwrap();+ let mut config = xerj_common::config::Config::default();+ config.server.data_dir = dir.path().to_string_lossy().into_owned();+ config.limits.max_fields_per_index = 50;+ let engine_instance = Engine::new(config).expect("engine");+ engine_instance+ .create_index("field-cap", Schema::empty())+ .unwrap();+ let index = engine_instance.get_index("field-cap").unwrap();++ // Fill exactly to the cap (robust to whatever base fields exist).+ let base = index.schema().await.field_count() as u32;+ let fill: Vec<FieldConfig> = (0..(50 - base))+ .map(|i| FieldConfig::new(format!("f{i}"), FieldType::Keyword))+ .collect();+ index+ .add_fields(fill)+ .await+ .expect("filling exactly to the cap must succeed");++ // One field over the cap via the explicit path must be REFUSED.+ let err = index+ .add_fields(vec![FieldConfig::new("overflow", FieldType::Keyword)])+ .await+ .expect_err("exceeding max_fields_per_index on the explicit path must be rejected");+ assert!(+ err.to_string().contains("Limit of total fields"),+ "unexpected error: {err}"+ );+ }+#[tokio::test]async fn mapping_vector_over_existing_hidden_chunks_fails_closed_before_visibility() {let dir = TempDir::new().unwrap();@@ -8526,18 +8564,27 @@ impl Index {// useful cross-document scheduling window without giant padding.let max_passages_per_window = self.embedding_config.onnx_scheduling_window;let passage_counts = jobs.iter().map(|job| job.texts.len()).collect::<Vec<_>>();+ // Window boundaries only — do NOT materialize passage texts here.+ // Pre-building (and cloning) every window's texts up front held+ // ~N× the request's semantic text for the entire embedding loop+ // (#71). Keep `(start, end, passages)` and build each window's texts+ // lazily, MOVING them into `embed_batch`, so only the in-flight+ // window(s) are resident at a time.let mut windows = Vec::new();let mut start = 0;while start < jobs.len() {let (end, passages) =semantic_embedding_window_end(&passage_counts, start, max_passages_per_window);- let texts: Vec<String> = jobs[start..end]- .iter()- .flat_map(|job| job.texts.iter().cloned())- .collect();- windows.push((start, end, passages, texts));+ windows.push((start, end, passages));start = end;}+ // Collect one window's passage texts on demand (borrows `jobs`).+ let window_texts = |start: usize, end: usize| -> Vec<String> {+ jobs[start..end]+ .iter()+ .flat_map(|job| job.texts.iter().cloned())+ .collect()+ };// Two sessions are an explicit opt-in. Launch at most two complete// windows together, drain both, then consume results in input@@ -8549,7 +8596,8 @@ impl Index {self.embedding_config.onnx_session_pool_size,) {let results = collect_ordinal_buffered_two(windows.len(), |ordinal| {- embedder.embed_batch(windows[ordinal].3.clone())+ let (start, end, _passages) = windows[ordinal];+ embedder.embed_batch(window_texts(start, end))}).await;window_results.extend(@@ -8559,12 +8607,12 @@ impl Index {.map(|(window, result)| (window.0, window.1, window.2, result)),);} else {- for (start, end, passages, texts) in &windows {+ for &(start, end, passages) in &windows {window_results.push((- *start,- *end,- *passages,- embedder.embed_batch(texts.clone()).await,+ start,+ end,+ passages,+ embedder.embed_batch(window_texts(start, end)).await,));}}@@ -14199,6 +14247,22 @@ impl Index {for field in fields {candidate.add_field(field)?;}+ // Mapping-explosion guard (#76 S5-5): enforce `max_fields_per_index` on+ // the EXPLICIT mapping path (`PUT /:index/_mapping`,+ // `POST /v1/schema/:name/evolve`) too, not only the dynamic-ingest path.+ // `ManagedSchema::add_field` does not check the cap, so without this an+ // authenticated caller could bypass the mapping-explosion limit by+ // supplying fields explicitly. Reject before publishing the candidate.+ let projected = candidate.schema.field_count() as u32;+ if projected > self.max_fields_per_index {+ return Err(EngineError::Common(+ xerj_common::XerjError::invalid_mapping(format!(+ "Limit of total fields [{}] has been exceeded: this mapping update would \+ bring the index to {projected} fields",+ self.max_fields_per_index+ )),+ ));+ }if self.live_doc_count() > 0 {let active_before: HashSet<String> = passage_scored_vector_fields(&schema.schema).into_iter()