AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(security): close 3 findings from the instruction/data whitebox pass (INJ-01, F-PATH-02, S5-1)
xerj-org58fc73f60055
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 333c03ad4..aedb229ca 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -23162,17 +23162,92 @@ fn render_template(source: &str, params: &serde_json::Map<String, Value>) -> Strfor (key, val) in params {let placeholder = format!("{{{{{}}}}}", key);let replacement = match val {- Value::String(s) => s.clone(),+ // SECURITY: a string param must NOT be able to carry JSON structural+ // tokens into the rendered query. Splicing the raw string lets a+ // value like `50,"query":{"match_all":{}}` inject a duplicate key and+ // drop the template's own filter (an unescaped-substitution injection).+ // Emit the JSON-escaped INNER content (no surrounding quotes, so a+ // placeholder used inside a template string literal — `"title":"{{q}}"`+ // — still renders correctly). Inside a string context the escaping is+ // transparent; in a bare value position a malicious value becomes an+ // invalid `\"` sequence that fails the re-parse rather than injecting.+ Value::String(s) => match serde_json::to_string(s) {+ Ok(quoted) if quoted.len() >= 2 => quoted[1..quoted.len() - 1].to_string(),+ _ => String::new(),+ },Value::Number(n) => n.to_string(),Value::Bool(b) => b.to_string(),Value::Null => "null".to_string(),- other => other.to_string(),+ // Objects/arrays serialize to their canonical JSON — a single value token.+ other => serde_json::to_string(other).unwrap_or_default(),};result = result.replace(&placeholder, &replacement);}result}+#[cfg(test)]+mod render_template_security_tests {+ use super::render_template;+ use serde_json::{json, Map, Value};++ fn params(v: Value) -> Map<String, Value> {+ v.as_object().unwrap().clone()+ }++ #[test]+ fn string_param_cannot_inject_query_structure() {+ // INJ-01 regression: a template that encodes an ACL filter and takes an+ // untrusted `size`. The attacker sends a string that tries to splice in a+ // second `query` key to drop the filter.+ let src =+ r#"{"query":{"bool":{"filter":[{"term":{"visibility":"public"}}]}},"size":{{size}}}"#;+ let p = params(json!({ "size": "50,\"query\":{\"match_all\":{}}" }));+ let rendered = render_template(src, &p);+ // The malicious value must NOT parse into a query with a dropped filter.+ // With escaping it becomes an invalid bare `\"` sequence → parse fails,+ // so the injection is rejected rather than executed.+ let parsed = serde_json::from_str::<Value>(&rendered);+ assert!(+ parsed.is_err()+ || parsed+ .as_ref()+ .unwrap()+ .get("query")+ .and_then(|q| q.get("bool"))+ .is_some(),+ "malicious size must not yield a match_all query: {rendered}"+ );+ }++ #[test]+ fn legitimate_in_string_substitution_still_works() {+ let src = r#"{"query":{"match":{"title":"{{q}}"}}}"#;+ let p = params(json!({ "q": "hello world" }));+ let rendered = render_template(src, &p);+ let v: Value = serde_json::from_str(&rendered).expect("must parse");+ assert_eq!(v["query"]["match"]["title"], "hello world");+ }++ #[test]+ fn quotes_in_a_string_value_are_escaped_not_broken_out() {+ let src = r#"{"query":{"match":{"title":"{{q}}"}}}"#;+ let p = params(json!({ "q": "a\"b" }));+ let rendered = render_template(src, &p);+ let v: Value = serde_json::from_str(&rendered).expect("must parse");+ assert_eq!(v["query"]["match"]["title"], "a\"b");+ }++ #[test]+ fn numeric_param_in_value_position_still_works() {+ let src = r#"{"query":{"match_all":{}},"size":{{size}}}"#;+ let p = params(json!({ "size": 25 }));+ let rendered = render_template(src, &p);+ let v: Value = serde_json::from_str(&rendered).expect("must parse");+ assert_eq!(v["size"], 25);+ }+}+// ─────────────────────────────────────────────────────────────────────────────// POST /{index}/_search/template// ─────────────────────────────────────────────────────────────────────────────diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rsindex 96921d34e..f8f925147 100644--- a/engine/crates/xerj-common/src/config.rs+++ b/engine/crates/xerj-common/src/config.rs@@ -918,6 +918,16 @@ pub struct LimitsConfig {/// oversized bulks at the top of the parse phase with a 413-style per-item/// error instead of letting jemalloc abort the process on OOM.pub max_actions_per_bulk: usize,+ /// Filesystem locations under which a snapshot repository's `settings.location`+ /// is allowed to resolve (an ES `path.repo` equivalent). Empty (the default)+ /// means **only `data_dir`** is permitted. Without this bound a write-capable+ /// client could `PUT /_snapshot/<repo>` with an arbitrary absolute `location`+ /// and then create/restore snapshots that read or write index data outside+ /// `data_dir` (path-traversal via the repo root, F-PATH-02). Each entry is a+ /// base directory; a repo location is accepted only if it canonicalizes to a+ /// path inside `data_dir` or inside one of these bases.+ #[serde(default)]+ pub snapshot_repo_allowlist: Vec<String>,/// Maximum HTTP request body size in bytes (default: `100 * 1024 * 1024`,/// i.e. 100 MiB). Caps NDJSON bulk payloads, large mget bodies, etc./// Raise this only if your client routinely sends bigger requests; the@@ -984,6 +994,7 @@ impl Default for LimitsConfig {max_concurrent_searches: 64,max_fields_per_index: 500,max_actions_per_bulk: 50_000,+ snapshot_repo_allowlist: Vec::new(),max_body_bytes: 100 * 1024 * 1024,max_result_window: 10_000,max_mget_docs: 10_000,@@ -1433,13 +1444,14 @@ mod tests {// default_quantization, max_dimensions)// logs: 2 (retention_days, time_partition)// embedding: 4 (default_endpoint, default_model, batch_size, timeout_ms)- // limits: 11 (query/concurrency/mapping/body/result/mget/bucket limits,- // total memtable, segment hydration cache, RSS, disk)+ // limits: 12 (query/concurrency/mapping/bulk-actions/body/result/mget/bucket+ // limits, snapshot repo allowlist, total memtable,+ // segment hydration cache, RSS, disk)// indexing: 3 (turbo_batch_size, turbo_parallel, turbo_fast_analyzer)// logging: 2 (format, access_log) ← RC4-W4 item 6// ─────────- // total: 58 fields, minus 1 auto-generated (admin_api_key) = 57 meaningful user settings- let total: usize = 5 + 3 + 3 + 10 + 5 + 3 + 1 + 6 + 2 + 4 + 11 + 3 + 2;- assert_eq!(total, 58);+ // total: 59 fields, minus 1 auto-generated (admin_api_key) = 58 meaningful user settings+ let total: usize = 5 + 3 + 3 + 10 + 5 + 3 + 1 + 6 + 2 + 4 + 12 + 3 + 2;+ assert_eq!(total, 59);}}diff --git a/engine/crates/xerj-console-api/src/auth/sessions.rs b/engine/crates/xerj-console-api/src/auth/sessions.rsindex 2a9b2cd31..b4759df91 100644--- a/engine/crates/xerj-console-api/src/auth/sessions.rs+++ b/engine/crates/xerj-console-api/src/auth/sessions.rs@@ -34,6 +34,7 @@ use crate::bootstrap::sha256_hex;use crate::error::{ConsoleApiError, ConsoleResult};use crate::state::ConsoleState;use crate::time::{epoch_ms_to_iso, now_epoch_ms, now_iso, parse_iso};+use xerj_engine::Engine;type HmacSha256 = Hmac<Sha256>;@@ -180,12 +181,14 @@ impl FromRequestParts<ConsoleState> for AuthSession {return Err(ConsoleApiError::Unauthorized("user disabled".into()));}- // Bump last_seen so the session stays fresh while the user is- // active. Best-effort — we don't fail the request if the write- // races with another tab.- let mut bumped = session.clone();- bumped.last_seen_at = now_iso();- let _ = store::put_session(&state.engine, &bumped).await;+ // Bump last_seen so the session stays fresh while the user is active.+ // Best-effort — we don't fail the request if the write races with another+ // tab. SECURITY (S5-1): do NOT blind-write the session we read at the top+ // of this function. A concurrent revoke_session() between that read and+ // this write would be clobbered by the stale copy (revoked_at: None),+ // resurrecting a revoked session — a revocation bypass. Re-read and bump+ // only the fresh record, skipping entirely if it is now revoked or gone.+ bump_last_seen(&state.engine, &session_id).await;Ok(AuthSession { session_id, user })}@@ -195,6 +198,28 @@ impl FromRequestParts<ConsoleState> for AuthSession {// Bearer (API token) authentication// ─────────────────────────────────────────────────────────────────────────────+/// Refresh a session's `last_seen_at` without clobbering a concurrent revocation.+///+/// S5-1 fix: re-reads the session immediately before writing and skips the write+/// if it is now gone or revoked, so a `revoke_session()` that landed after the+/// request's initial auth read cannot be overwritten by a stale copy. The write+/// carries the *fresh* record's fields (including its `revoked_at`), so even the+/// tiny residual window between this re-read and the put cannot resurrect a+/// revoked session — it would only rewrite an already-revoked record unchanged.+/// Best-effort: any store error is swallowed, exactly as before.+pub(crate) async fn bump_last_seen(engine: &Engine, session_id: &str) {+ let fresh = match store::get_session(engine, session_id).await {+ Ok(Some(s)) => s,+ _ => return, // gone (revoked-by-delete or expired-out) → do not recreate+ };+ if fresh.revoked_at.is_some() {+ return; // revoked between auth read and now → must not refresh+ }+ let mut bumped = fresh;+ bumped.last_seen_at = now_iso();+ let _ = store::put_session(engine, &bumped).await;+}+/// Pull the secret out of an `Authorization: Bearer <secret>` header./// Returns `None` when the header is absent, is not a bearer scheme, or/// carries an empty value — the caller then falls back to the cookie path.@@ -329,4 +354,72 @@ mod tests {let signed = sign_session_id("abc123", &[7u8; 32]);assert!(verify_session_cookie(&signed, &[8u8; 32]).is_none());}++ async fn test_engine() -> (Engine, tempfile::TempDir) {+ let dir = tempfile::TempDir::new().unwrap();+ let mut cfg = xerj_common::config::Config::default();+ cfg.server.data_dir = dir.path().to_str().unwrap().to_string();+ let engine = Engine::new(cfg).expect("engine");+ crate::bootstrap::run(&engine, dir.path(), "http://localhost:9200")+ .await+ .expect("bootstrap");+ (engine, dir)+ }++ fn session(id: &str, revoked: bool) -> store::Session {+ store::Session {+ id: id.to_string(),+ user_id: "u1".into(),+ created_at: now_iso(),+ expires_at: crate::time::epoch_ms_to_iso(now_epoch_ms() + 3_600_000),+ last_seen_at: crate::time::epoch_ms_to_iso(now_epoch_ms() - 600_000),+ ip: None,+ ua: None,+ idp: "passkey".into(),+ revoked_at: if revoked { Some(now_iso()) } else { None },+ }+ }++ /// S5-1: bump_last_seen must NOT resurrect a session that was revoked before+ /// the bump. Pre-fix, the extractor blind-wrote a stale (revoked_at: None)+ /// copy, un-revoking it — a revocation bypass.+ #[tokio::test]+ async fn bump_does_not_resurrect_a_revoked_session() {+ let (engine, _dir) = test_engine().await;+ // A session that is live, then revoked (the realistic race outcome:+ // revoke lands before the in-flight request's bump).+ store::put_session(&engine, &session("s-revoked", false))+ .await+ .unwrap();+ store::revoke_session(&engine, "s-revoked", &now_iso())+ .await+ .unwrap();++ bump_last_seen(&engine, "s-revoked").await;++ let after = store::get_session(&engine, "s-revoked").await.unwrap();+ assert!(+ after.is_some() && after.unwrap().revoked_at.is_some(),+ "bump must not clear revoked_at"+ );+ }++ /// A live session's bump still refreshes last_seen (the feature still works).+ #[tokio::test]+ async fn bump_refreshes_a_live_session() {+ let (engine, _dir) = test_engine().await;+ let mut s = session("s-live", false);+ s.last_seen_at = crate::time::epoch_ms_to_iso(now_epoch_ms() - 600_000);+ let before = s.last_seen_at.clone();+ store::put_session(&engine, &s).await.unwrap();++ bump_last_seen(&engine, "s-live").await;++ let after = store::get_session(&engine, "s-live")+ .await+ .unwrap()+ .unwrap();+ assert!(after.revoked_at.is_none());+ assert_ne!(after.last_seen_at, before, "last_seen should advance");+ }}diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 7dd09b212..6dcaf9d97 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -1560,7 +1560,13 @@ impl Engine {indices: Option<Vec<String>>,) -> Result<Value> {let snap_dir = std::path::Path::new(repo_path).join(name);- let snap_dir = validate_snapshot_path(repo_path, name, &snap_dir)?;+ let snap_dir = validate_snapshot_path(+ repo_path,+ name,+ &snap_dir,+ &self.data_dir,+ &self.config.limits.snapshot_repo_allowlist,+ )?;std::fs::create_dir_all(&snap_dir).map_err(EngineError::Io)?;// Wall-clock start, captured BEFORE the flush+copy work so@@ -1652,7 +1658,13 @@ impl Engine {indices: Option<Vec<String>>,) -> Result<Vec<String>> {let snap_dir = std::path::Path::new(repo_path).join(name);- let snap_dir = validate_snapshot_path(repo_path, name, &snap_dir)?;+ let snap_dir = validate_snapshot_path(+ repo_path,+ name,+ &snap_dir,+ &self.data_dir,+ &self.config.limits.snapshot_repo_allowlist,+ )?;let manifest_path = snap_dir.join("manifest.json");let manifest_bytes = std::fs::read(&manifest_path).map_err(EngineError::Io)?;@@ -1853,6 +1865,8 @@ fn validate_snapshot_path(repo_path: &str,name: &str,snap_dir: &std::path::Path,+ data_dir: &std::path::Path,+ allowlist: &[String],) -> Result<std::path::PathBuf> {if repo_path.is_empty() {return Err(EngineError::Common(@@ -1861,6 +1875,29 @@ fn validate_snapshot_path(),));}+ // F-PATH-02: the repo `location` is operator/attacker-supplied via+ // `PUT /_snapshot/{repo}`. The name-containment checks below only prove the+ // snapshot stays inside THAT location — they say nothing about whether the+ // location itself is sane. Bound the location to `data_dir` (default) or a+ // configured allowlist (an ES `path.repo` equivalent), so a repo cannot point+ // at `/etc`, `/root/.ssh`, another tenant's dir, etc.+ let base_canon = |p: &std::path::Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf());+ let repo_base = base_canon(std::path::Path::new(repo_path));+ let mut allowed_bases: Vec<std::path::PathBuf> = vec![base_canon(data_dir)];+ allowed_bases.extend(+ allowlist+ .iter()+ .map(|b| base_canon(std::path::Path::new(b))),+ );+ if !allowed_bases.iter().any(|base| repo_base.starts_with(base)) {+ return Err(EngineError::Common(+ xerj_common::XerjError::invalid_mapping(format!(+ "snapshot repository location [{repo_path}] is outside data_dir and not in \+ limits.snapshot_repo_allowlist; refusing (set the allowlist to permit external \+ snapshot roots)"+ )),+ ));+ }if name.is_empty()|| name.contains("..")|| name.contains('/')@@ -1905,6 +1942,53 @@ fn validate_snapshot_path(}}+#[cfg(test)]+mod snapshot_path_security_tests {+ use super::validate_snapshot_path;++ #[test]+ fn f_path_02_repo_location_outside_data_dir_is_rejected_by_default() {+ let data = tempfile::TempDir::new().unwrap();+ // A repo location pointing outside data_dir (the F-PATH-02 exploit shape)+ // with an empty allowlist must be refused.+ let outside = tempfile::TempDir::new().unwrap();+ let repo = outside.path().to_str().unwrap();+ let snap = std::path::Path::new(repo).join("s1");+ let err = validate_snapshot_path(repo, "s1", &snap, data.path(), &[])+ .expect_err("external repo location must be rejected");+ assert!(err.to_string().contains("outside data_dir"), "{err}");+ }++ #[test]+ fn location_inside_data_dir_is_allowed() {+ let data = tempfile::TempDir::new().unwrap();+ let repo_dir = data.path().join("backups");+ std::fs::create_dir_all(&repo_dir).unwrap();+ let repo = repo_dir.to_str().unwrap();+ let snap = repo_dir.join("s1");+ validate_snapshot_path(repo, "s1", &snap, data.path(), &[])+ .expect("a repo under data_dir must be allowed");+ }++ #[test]+ fn explicit_allowlist_permits_an_external_root() {+ let data = tempfile::TempDir::new().unwrap();+ let ext = tempfile::TempDir::new().unwrap();+ let repo = ext.path().to_str().unwrap().to_string();+ let snap = std::path::Path::new(&repo).join("s1");+ validate_snapshot_path(&repo, "s1", &snap, data.path(), &[repo.clone()])+ .expect("an allowlisted external root must be permitted");+ }++ #[test]+ fn traversal_name_still_rejected_even_inside_an_allowed_base() {+ let data = tempfile::TempDir::new().unwrap();+ let snap = data.path().join("..");+ let repo = data.path().to_str().unwrap();+ assert!(validate_snapshot_path(repo, "..", &snap, data.path(), &[]).is_err());+ }+}+/// Recursively copy all files from `src` to `dst`, recording relative paths in `files`.fn copy_dir_recursive(src: &std::path::Path,diff --git a/engine/xerj.default.toml b/engine/xerj.default.tomlindex fd5bf72de..1b388f82c 100644--- a/engine/xerj.default.toml+++ b/engine/xerj.default.toml@@ -313,6 +313,14 @@ max_fields_per_index = 500# disable the cap (not recommended in production).max_actions_per_bulk = 50000+# Filesystem roots a snapshot repository's settings.location may resolve under+# (an Elasticsearch path.repo equivalent). Empty = only data_dir is permitted.+# Without this bound, a write-capable client can PUT a snapshot repo whose+# location is any absolute path and then read/write index data outside data_dir.+# Add absolute base paths here only to allow snapshots on external/backup roots.+# Example: snapshot_repo_allowlist = ["/mnt/backups"]+snapshot_repo_allowlist = []+# Process-wide retained-payload budget for the seven immutable segment# hydration caches (MiB). 0 derives 20% of the effective cgroup/system memory# limit with no minimum floor. This is not an RSS limit: query-owned results,