Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(security): close 3 findings from the instruction/data whitebox pass (INJ-01, F-PATH-02, S5-1)

xerj-org58fc73f60055
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 333c03ad4..aedb229ca 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -23162,17 +23162,92 @@ fn render_template(source: &str, params: &serde_json::Map<String, Value>) -> Str
for (key, val) in params {
let placeholder = format!("{{{{{}}}}}", key);
let replacement = match val {
- Value::String(s) => s.clone(),
+ // SECURITY: a string param must NOT be able to carry JSON structural
+ // tokens into the rendered query. Splicing the raw string lets a
+ // value like `50,"query":{"match_all":{}}` inject a duplicate key and
+ // drop the template's own filter (an unescaped-substitution injection).
+ // Emit the JSON-escaped INNER content (no surrounding quotes, so a
+ // placeholder used inside a template string literal — `"title":"{{q}}"`
+ // — still renders correctly). Inside a string context the escaping is
+ // transparent; in a bare value position a malicious value becomes an
+ // invalid `\"` sequence that fails the re-parse rather than injecting.
+ Value::String(s) => match serde_json::to_string(s) {
+ Ok(quoted) if quoted.len() >= 2 => quoted[1..quoted.len() - 1].to_string(),
+ _ => String::new(),
+ },
Value::Number(n) => n.to_string(),
Value::Bool(b) => b.to_string(),
Value::Null => "null".to_string(),
- other => other.to_string(),
+ // Objects/arrays serialize to their canonical JSON — a single value token.
+ other => serde_json::to_string(other).unwrap_or_default(),
};
result = result.replace(&placeholder, &replacement);
}
result
}
+#[cfg(test)]
+mod render_template_security_tests {
+ use super::render_template;
+ use serde_json::{json, Map, Value};
+
+ fn params(v: Value) -> Map<String, Value> {
+ v.as_object().unwrap().clone()
+ }
+
+ #[test]
+ fn string_param_cannot_inject_query_structure() {
+ // INJ-01 regression: a template that encodes an ACL filter and takes an
+ // untrusted `size`. The attacker sends a string that tries to splice in a
+ // second `query` key to drop the filter.
+ let src =
+ r#"{"query":{"bool":{"filter":[{"term":{"visibility":"public"}}]}},"size":{{size}}}"#;
+ let p = params(json!({ "size": "50,\"query\":{\"match_all\":{}}" }));
+ let rendered = render_template(src, &p);
+ // The malicious value must NOT parse into a query with a dropped filter.
+ // With escaping it becomes an invalid bare `\"` sequence → parse fails,
+ // so the injection is rejected rather than executed.
+ let parsed = serde_json::from_str::<Value>(&rendered);
+ assert!(
+ parsed.is_err()
+ || parsed
+ .as_ref()
+ .unwrap()
+ .get("query")
+ .and_then(|q| q.get("bool"))
+ .is_some(),
+ "malicious size must not yield a match_all query: {rendered}"
+ );
+ }
+
+ #[test]
+ fn legitimate_in_string_substitution_still_works() {
+ let src = r#"{"query":{"match":{"title":"{{q}}"}}}"#;
+ let p = params(json!({ "q": "hello world" }));
+ let rendered = render_template(src, &p);
+ let v: Value = serde_json::from_str(&rendered).expect("must parse");
+ assert_eq!(v["query"]["match"]["title"], "hello world");
+ }
+
+ #[test]
+ fn quotes_in_a_string_value_are_escaped_not_broken_out() {
+ let src = r#"{"query":{"match":{"title":"{{q}}"}}}"#;
+ let p = params(json!({ "q": "a\"b" }));
+ let rendered = render_template(src, &p);
+ let v: Value = serde_json::from_str(&rendered).expect("must parse");
+ assert_eq!(v["query"]["match"]["title"], "a\"b");
+ }
+
+ #[test]
+ fn numeric_param_in_value_position_still_works() {
+ let src = r#"{"query":{"match_all":{}},"size":{{size}}}"#;
+ let p = params(json!({ "size": 25 }));
+ let rendered = render_template(src, &p);
+ let v: Value = serde_json::from_str(&rendered).expect("must parse");
+ assert_eq!(v["size"], 25);
+ }
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// POST /{index}/_search/template
// ─────────────────────────────────────────────────────────────────────────────
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 96921d34e..f8f925147 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -918,6 +918,16 @@ pub struct LimitsConfig {
/// oversized bulks at the top of the parse phase with a 413-style per-item
/// error instead of letting jemalloc abort the process on OOM.
pub max_actions_per_bulk: usize,
+ /// Filesystem locations under which a snapshot repository's `settings.location`
+ /// is allowed to resolve (an ES `path.repo` equivalent). Empty (the default)
+ /// means **only `data_dir`** is permitted. Without this bound a write-capable
+ /// client could `PUT /_snapshot/<repo>` with an arbitrary absolute `location`
+ /// and then create/restore snapshots that read or write index data outside
+ /// `data_dir` (path-traversal via the repo root, F-PATH-02). Each entry is a
+ /// base directory; a repo location is accepted only if it canonicalizes to a
+ /// path inside `data_dir` or inside one of these bases.
+ #[serde(default)]
+ pub snapshot_repo_allowlist: Vec<String>,
/// Maximum HTTP request body size in bytes (default: `100 * 1024 * 1024`,
/// i.e. 100 MiB). Caps NDJSON bulk payloads, large mget bodies, etc.
/// Raise this only if your client routinely sends bigger requests; the
@@ -984,6 +994,7 @@ impl Default for LimitsConfig {
max_concurrent_searches: 64,
max_fields_per_index: 500,
max_actions_per_bulk: 50_000,
+ snapshot_repo_allowlist: Vec::new(),
max_body_bytes: 100 * 1024 * 1024,
max_result_window: 10_000,
max_mget_docs: 10_000,
@@ -1433,13 +1444,14 @@ mod tests {
// default_quantization, max_dimensions)
// logs: 2 (retention_days, time_partition)
// embedding: 4 (default_endpoint, default_model, batch_size, timeout_ms)
- // limits: 11 (query/concurrency/mapping/body/result/mget/bucket limits,
- // total memtable, segment hydration cache, RSS, disk)
+ // limits: 12 (query/concurrency/mapping/bulk-actions/body/result/mget/bucket
+ // limits, snapshot repo allowlist, total memtable,
+ // segment hydration cache, RSS, disk)
// indexing: 3 (turbo_batch_size, turbo_parallel, turbo_fast_analyzer)
// logging: 2 (format, access_log) ← RC4-W4 item 6
// ─────────
- // total: 58 fields, minus 1 auto-generated (admin_api_key) = 57 meaningful user settings
- let total: usize = 5 + 3 + 3 + 10 + 5 + 3 + 1 + 6 + 2 + 4 + 11 + 3 + 2;
- assert_eq!(total, 58);
+ // total: 59 fields, minus 1 auto-generated (admin_api_key) = 58 meaningful user settings
+ let total: usize = 5 + 3 + 3 + 10 + 5 + 3 + 1 + 6 + 2 + 4 + 12 + 3 + 2;
+ assert_eq!(total, 59);
}
}
diff --git a/engine/crates/xerj-console-api/src/auth/sessions.rs b/engine/crates/xerj-console-api/src/auth/sessions.rs
index 2a9b2cd31..b4759df91 100644
--- a/engine/crates/xerj-console-api/src/auth/sessions.rs
+++ b/engine/crates/xerj-console-api/src/auth/sessions.rs
@@ -34,6 +34,7 @@ use crate::bootstrap::sha256_hex;
use crate::error::{ConsoleApiError, ConsoleResult};
use crate::state::ConsoleState;
use crate::time::{epoch_ms_to_iso, now_epoch_ms, now_iso, parse_iso};
+use xerj_engine::Engine;
type HmacSha256 = Hmac<Sha256>;
@@ -180,12 +181,14 @@ impl FromRequestParts<ConsoleState> for AuthSession {
return Err(ConsoleApiError::Unauthorized("user disabled".into()));
}
- // Bump last_seen so the session stays fresh while the user is
- // active. Best-effort — we don't fail the request if the write
- // races with another tab.
- let mut bumped = session.clone();
- bumped.last_seen_at = now_iso();
- let _ = store::put_session(&state.engine, &bumped).await;
+ // Bump last_seen so the session stays fresh while the user is active.
+ // Best-effort — we don't fail the request if the write races with another
+ // tab. SECURITY (S5-1): do NOT blind-write the session we read at the top
+ // of this function. A concurrent revoke_session() between that read and
+ // this write would be clobbered by the stale copy (revoked_at: None),
+ // resurrecting a revoked session — a revocation bypass. Re-read and bump
+ // only the fresh record, skipping entirely if it is now revoked or gone.
+ bump_last_seen(&state.engine, &session_id).await;
Ok(AuthSession { session_id, user })
}
@@ -195,6 +198,28 @@ impl FromRequestParts<ConsoleState> for AuthSession {
// Bearer (API token) authentication
// ─────────────────────────────────────────────────────────────────────────────
+/// Refresh a session's `last_seen_at` without clobbering a concurrent revocation.
+///
+/// S5-1 fix: re-reads the session immediately before writing and skips the write
+/// if it is now gone or revoked, so a `revoke_session()` that landed after the
+/// request's initial auth read cannot be overwritten by a stale copy. The write
+/// carries the *fresh* record's fields (including its `revoked_at`), so even the
+/// tiny residual window between this re-read and the put cannot resurrect a
+/// revoked session — it would only rewrite an already-revoked record unchanged.
+/// Best-effort: any store error is swallowed, exactly as before.
+pub(crate) async fn bump_last_seen(engine: &Engine, session_id: &str) {
+ let fresh = match store::get_session(engine, session_id).await {
+ Ok(Some(s)) => s,
+ _ => return, // gone (revoked-by-delete or expired-out) → do not recreate
+ };
+ if fresh.revoked_at.is_some() {
+ return; // revoked between auth read and now → must not refresh
+ }
+ let mut bumped = fresh;
+ bumped.last_seen_at = now_iso();
+ let _ = store::put_session(engine, &bumped).await;
+}
+
/// Pull the secret out of an `Authorization: Bearer <secret>` header.
/// Returns `None` when the header is absent, is not a bearer scheme, or
/// carries an empty value — the caller then falls back to the cookie path.
@@ -329,4 +354,72 @@ mod tests {
let signed = sign_session_id("abc123", &[7u8; 32]);
assert!(verify_session_cookie(&signed, &[8u8; 32]).is_none());
}
+
+ async fn test_engine() -> (Engine, tempfile::TempDir) {
+ let dir = tempfile::TempDir::new().unwrap();
+ let mut cfg = xerj_common::config::Config::default();
+ cfg.server.data_dir = dir.path().to_str().unwrap().to_string();
+ let engine = Engine::new(cfg).expect("engine");
+ crate::bootstrap::run(&engine, dir.path(), "http://localhost:9200")
+ .await
+ .expect("bootstrap");
+ (engine, dir)
+ }
+
+ fn session(id: &str, revoked: bool) -> store::Session {
+ store::Session {
+ id: id.to_string(),
+ user_id: "u1".into(),
+ created_at: now_iso(),
+ expires_at: crate::time::epoch_ms_to_iso(now_epoch_ms() + 3_600_000),
+ last_seen_at: crate::time::epoch_ms_to_iso(now_epoch_ms() - 600_000),
+ ip: None,
+ ua: None,
+ idp: "passkey".into(),
+ revoked_at: if revoked { Some(now_iso()) } else { None },
+ }
+ }
+
+ /// S5-1: bump_last_seen must NOT resurrect a session that was revoked before
+ /// the bump. Pre-fix, the extractor blind-wrote a stale (revoked_at: None)
+ /// copy, un-revoking it — a revocation bypass.
+ #[tokio::test]
+ async fn bump_does_not_resurrect_a_revoked_session() {
+ let (engine, _dir) = test_engine().await;
+ // A session that is live, then revoked (the realistic race outcome:
+ // revoke lands before the in-flight request's bump).
+ store::put_session(&engine, &session("s-revoked", false))
+ .await
+ .unwrap();
+ store::revoke_session(&engine, "s-revoked", &now_iso())
+ .await
+ .unwrap();
+
+ bump_last_seen(&engine, "s-revoked").await;
+
+ let after = store::get_session(&engine, "s-revoked").await.unwrap();
+ assert!(
+ after.is_some() && after.unwrap().revoked_at.is_some(),
+ "bump must not clear revoked_at"
+ );
+ }
+
+ /// A live session's bump still refreshes last_seen (the feature still works).
+ #[tokio::test]
+ async fn bump_refreshes_a_live_session() {
+ let (engine, _dir) = test_engine().await;
+ let mut s = session("s-live", false);
+ s.last_seen_at = crate::time::epoch_ms_to_iso(now_epoch_ms() - 600_000);
+ let before = s.last_seen_at.clone();
+ store::put_session(&engine, &s).await.unwrap();
+
+ bump_last_seen(&engine, "s-live").await;
+
+ let after = store::get_session(&engine, "s-live")
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(after.revoked_at.is_none());
+ assert_ne!(after.last_seen_at, before, "last_seen should advance");
+ }
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 7dd09b212..6dcaf9d97 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1560,7 +1560,13 @@ impl Engine {
indices: Option<Vec<String>>,
) -> Result<Value> {
let snap_dir = std::path::Path::new(repo_path).join(name);
- let snap_dir = validate_snapshot_path(repo_path, name, &snap_dir)?;
+ let snap_dir = validate_snapshot_path(
+ repo_path,
+ name,
+ &snap_dir,
+ &self.data_dir,
+ &self.config.limits.snapshot_repo_allowlist,
+ )?;
std::fs::create_dir_all(&snap_dir).map_err(EngineError::Io)?;
// Wall-clock start, captured BEFORE the flush+copy work so
@@ -1652,7 +1658,13 @@ impl Engine {
indices: Option<Vec<String>>,
) -> Result<Vec<String>> {
let snap_dir = std::path::Path::new(repo_path).join(name);
- let snap_dir = validate_snapshot_path(repo_path, name, &snap_dir)?;
+ let snap_dir = validate_snapshot_path(
+ repo_path,
+ name,
+ &snap_dir,
+ &self.data_dir,
+ &self.config.limits.snapshot_repo_allowlist,
+ )?;
let manifest_path = snap_dir.join("manifest.json");
let manifest_bytes = std::fs::read(&manifest_path).map_err(EngineError::Io)?;
@@ -1853,6 +1865,8 @@ fn validate_snapshot_path(
repo_path: &str,
name: &str,
snap_dir: &std::path::Path,
+ data_dir: &std::path::Path,
+ allowlist: &[String],
) -> Result<std::path::PathBuf> {
if repo_path.is_empty() {
return Err(EngineError::Common(
@@ -1861,6 +1875,29 @@ fn validate_snapshot_path(
),
));
}
+ // F-PATH-02: the repo `location` is operator/attacker-supplied via
+ // `PUT /_snapshot/{repo}`. The name-containment checks below only prove the
+ // snapshot stays inside THAT location — they say nothing about whether the
+ // location itself is sane. Bound the location to `data_dir` (default) or a
+ // configured allowlist (an ES `path.repo` equivalent), so a repo cannot point
+ // at `/etc`, `/root/.ssh`, another tenant's dir, etc.
+ let base_canon = |p: &std::path::Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf());
+ let repo_base = base_canon(std::path::Path::new(repo_path));
+ let mut allowed_bases: Vec<std::path::PathBuf> = vec![base_canon(data_dir)];
+ allowed_bases.extend(
+ allowlist
+ .iter()
+ .map(|b| base_canon(std::path::Path::new(b))),
+ );
+ if !allowed_bases.iter().any(|base| repo_base.starts_with(base)) {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_mapping(format!(
+ "snapshot repository location [{repo_path}] is outside data_dir and not in \
+ limits.snapshot_repo_allowlist; refusing (set the allowlist to permit external \
+ snapshot roots)"
+ )),
+ ));
+ }
if name.is_empty()
|| name.contains("..")
|| name.contains('/')
@@ -1905,6 +1942,53 @@ fn validate_snapshot_path(
}
}
+#[cfg(test)]
+mod snapshot_path_security_tests {
+ use super::validate_snapshot_path;
+
+ #[test]
+ fn f_path_02_repo_location_outside_data_dir_is_rejected_by_default() {
+ let data = tempfile::TempDir::new().unwrap();
+ // A repo location pointing outside data_dir (the F-PATH-02 exploit shape)
+ // with an empty allowlist must be refused.
+ let outside = tempfile::TempDir::new().unwrap();
+ let repo = outside.path().to_str().unwrap();
+ let snap = std::path::Path::new(repo).join("s1");
+ let err = validate_snapshot_path(repo, "s1", &snap, data.path(), &[])
+ .expect_err("external repo location must be rejected");
+ assert!(err.to_string().contains("outside data_dir"), "{err}");
+ }
+
+ #[test]
+ fn location_inside_data_dir_is_allowed() {
+ let data = tempfile::TempDir::new().unwrap();
+ let repo_dir = data.path().join("backups");
+ std::fs::create_dir_all(&repo_dir).unwrap();
+ let repo = repo_dir.to_str().unwrap();
+ let snap = repo_dir.join("s1");
+ validate_snapshot_path(repo, "s1", &snap, data.path(), &[])
+ .expect("a repo under data_dir must be allowed");
+ }
+
+ #[test]
+ fn explicit_allowlist_permits_an_external_root() {
+ let data = tempfile::TempDir::new().unwrap();
+ let ext = tempfile::TempDir::new().unwrap();
+ let repo = ext.path().to_str().unwrap().to_string();
+ let snap = std::path::Path::new(&repo).join("s1");
+ validate_snapshot_path(&repo, "s1", &snap, data.path(), &[repo.clone()])
+ .expect("an allowlisted external root must be permitted");
+ }
+
+ #[test]
+ fn traversal_name_still_rejected_even_inside_an_allowed_base() {
+ let data = tempfile::TempDir::new().unwrap();
+ let snap = data.path().join("..");
+ let repo = data.path().to_str().unwrap();
+ assert!(validate_snapshot_path(repo, "..", &snap, data.path(), &[]).is_err());
+ }
+}
+
/// Recursively copy all files from `src` to `dst`, recording relative paths in `files`.
fn copy_dir_recursive(
src: &std::path::Path,
diff --git a/engine/xerj.default.toml b/engine/xerj.default.toml
index fd5bf72de..1b388f82c 100644
--- a/engine/xerj.default.toml
+++ b/engine/xerj.default.toml
@@ -313,6 +313,14 @@ max_fields_per_index = 500
# disable the cap (not recommended in production).
max_actions_per_bulk = 50000
+# Filesystem roots a snapshot repository's settings.location may resolve under
+# (an Elasticsearch path.repo equivalent). Empty = only data_dir is permitted.
+# Without this bound, a write-capable client can PUT a snapshot repo whose
+# location is any absolute path and then read/write index data outside data_dir.
+# Add absolute base paths here only to allow snapshots on external/backup roots.
+# Example: snapshot_repo_allowlist = ["/mnt/backups"]
+snapshot_repo_allowlist = []
+
# Process-wide retained-payload budget for the seven immutable segment
# hydration caches (MiB). 0 derives 20% of the effective cgroup/system memory
# limit with no minimum floor. This is not an RSS limit: query-owned results,