Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

security: Phase-2 hardening — snapshot escape, field-limit, magic-link TOCTOU, info disclosure, index-name boundary; perf: ONNX windowing

xerj-org1c4e01ddac1d
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index e205beef9..77f820d27 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -675,6 +675,16 @@ pub async fn create_index(
return ApiError::new(e).into_response();
}
+ // #80: make the path-traversal guarantee LOCAL at this handler boundary.
+ // `Engine::create_index` also runs `IndexName::new`, but validating the name
+ // here means the es_compat surface itself rejects `..`, path separators, NUL
+ // and other malformed names up front — defense-in-depth, not a guarantee
+ // that lives only in a distant call. Same accepted set as before (the engine
+ // enforced the identical rule); the rejection is just local and obvious.
+ if let Err(e) = xerj_common::IndexName::validate(&index) {
+ return ApiError::new(e).into_response();
+ }
+
// Validate mapping field types before creating — ES returns
// mapper_parsing_exception for unsupported types.
let mappings_val = body.get("mappings").cloned().unwrap_or(Value::Null);
diff --git a/engine/crates/xerj-common/src/types.rs b/engine/crates/xerj-common/src/types.rs
index 905552061..985ded709 100644
--- a/engine/crates/xerj-common/src/types.rs
+++ b/engine/crates/xerj-common/src/types.rs
@@ -93,7 +93,11 @@ impl IndexName {
&self.0
}
- fn validate(name: &str) -> Result<(), XerjError> {
+ /// Validate an index name against the traversal/charset rules without
+ /// constructing an `IndexName`. Public so handler boundaries (e.g. the
+ /// es_compat `create_index` surface, #80) can reject `..`/separators/NUL
+ /// locally instead of relying solely on the deeper `IndexName::new` guard.
+ pub fn validate(name: &str) -> Result<(), XerjError> {
if name.len() > 255 {
return Err(XerjError::invalid_mapping(format!(
"index name too long: {} chars (max 255)",
diff --git a/engine/crates/xerj-console-api/src/auth/magic.rs b/engine/crates/xerj-console-api/src/auth/magic.rs
index ed6df65fe..cc024ce49 100644
--- a/engine/crates/xerj-console-api/src/auth/magic.rs
+++ b/engine/crates/xerj-console-api/src/auth/magic.rs
@@ -207,6 +207,14 @@ pub async fn redeem(
}
let token_hash = sha256_hex(body.token.as_bytes());
+ // #76 S5-3: serialize the single-use check and the mark-used commit. The
+ // used-check (below) and `mark_magic_link_used` (get→delete→create) are
+ // separated by several awaits with no exclusive lock, so two concurrent
+ // redeems of one token could both pass the check and both mint a session.
+ // Hold the gate for the rest of this function so check→consume is atomic.
+ // `lock_owned` avoids tying the guard's lifetime to `state` across awaits.
+ let _redeem_guard = state.redeem_gate.clone().lock_owned().await;
+
// Look it up.
let link = store::get_magic_link(&state.engine, &token_hash)
.await?
diff --git a/engine/crates/xerj-console-api/src/cluster.rs b/engine/crates/xerj-console-api/src/cluster.rs
index d89233d6a..deac26946 100644
--- a/engine/crates/xerj-console-api/src/cluster.rs
+++ b/engine/crates/xerj-console-api/src/cluster.rs
@@ -28,12 +28,42 @@ pub async fn info(State(state): State<ConsoleState>) -> ConsoleResult<Response>
ClusterMode::Raft => "raft",
};
- let body = json!({
+ Ok(ok(info_body(mode, state.started_at.0), None))
+}
+
+/// Build the unauthenticated `cluster/info` body.
+///
+/// AUTHZ-2: this endpoint is unauthenticated (the SPA reads it at boot, before
+/// login, to choose standalone-vs-topology view). It must therefore expose only
+/// what that decision needs — `mode` — plus the benign uptime. `node_id` and
+/// the exact build `version` are deliberately omitted: a pre-auth version string
+/// hands an attacker a precise target for known-CVE matching, and the node
+/// identity leaks cluster topology. Authenticated callers get the full detail
+/// from the credentialed cluster/status paths.
+fn info_body(mode: &str, started_at_ms: i64) -> serde_json::Value {
+ json!({
"mode": mode,
- "node_id": state.node_id.as_str(),
- "version": env!("CARGO_PKG_VERSION"),
- "started_at_ms": state.started_at.0,
- });
+ "started_at_ms": started_at_ms,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::info_body;
- Ok(ok(body, None))
+ #[test]
+ fn info_body_omits_node_id_and_version_pre_auth() {
+ let v = info_body("standalone", 1_700_000_000_000);
+ assert_eq!(v.get("mode").and_then(|m| m.as_str()), Some("standalone"));
+ assert!(v.get("started_at_ms").is_some());
+ // The AUTHZ-2 disclosure fields must be gone from the pre-auth body.
+ assert!(v.get("node_id").is_none(), "node_id must not leak pre-auth");
+ assert!(v.get("version").is_none(), "version must not leak pre-auth");
+ // And the whole serialized body must not contain the build version.
+ let s = v.to_string();
+ assert!(
+ !s.contains(env!("CARGO_PKG_VERSION")),
+ "build version must not appear in the pre-auth body: {s}"
+ );
+ }
}
diff --git a/engine/crates/xerj-console-api/src/state.rs b/engine/crates/xerj-console-api/src/state.rs
index 1a836b6ed..932f1a3f3 100644
--- a/engine/crates/xerj-console-api/src/state.rs
+++ b/engine/crates/xerj-console-api/src/state.rs
@@ -118,6 +118,15 @@ pub struct ConsoleState {
/// `data_dir/.xerj_master_key` file or the `XERJ_CONSOLE_KEY`
/// env var; generated on first start and persisted).
pub master_key: Arc<[u8; 32]>,
+
+ /// Serializes magic-link redemption (#76 S5-3). The single-use check and
+ /// the `mark_magic_link_used` commit are separated by several awaits and
+ /// the store's mark-used is itself get→delete→create, so two concurrent
+ /// redeems of one token could both pass the used-check and both mint a
+ /// session. Holding this gate across the whole check→consume makes the
+ /// transition atomic. Redemptions are rare (enrollment/recovery), so a
+ /// single gate is cheaper than a per-token map and needs no cleanup.
+ pub redeem_gate: Arc<tokio::sync::Mutex<()>>,
}
#[derive(Debug, Clone, Copy)]
@@ -159,6 +168,7 @@ impl ConsoleState {
enrollment_sessions: Arc::new(DashMap::new()),
auth_rate_counters: Arc::new(DashMap::new()),
master_key: Arc::new(master_key),
+ redeem_gate: Arc::new(tokio::sync::Mutex::new(())),
}
}
}
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 76cd7084b..36e788ef7 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -1881,6 +1881,24 @@ fn validate_snapshot_path(
// location itself is sane. Bound the location to `data_dir` (default) or a
// configured allowlist (an ES `path.repo` equivalent), so a repo cannot point
// at `/etc`, `/root/.ssh`, another tenant's dir, etc.
+ // F-PATH-02 (residual): a location whose fully-resolved target does not yet
+ // exist makes `canonicalize()` below fail, so the lexical fallback keeps any
+ // `..` components. A component-based `starts_with` then treats
+ // `<data_dir>/../../escape` as *inside* `<data_dir>` (it never normalizes
+ // `..`), and `create_dir_all` lets the OS walk out. Reject `..` outright —
+ // no legitimate snapshot root needs parent-dir traversal — so neither the
+ // location containment check nor the snapshot-name check can be bypassed
+ // with an unresolved parent reference.
+ if std::path::Path::new(repo_path)
+ .components()
+ .any(|c| matches!(c, std::path::Component::ParentDir))
+ {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_mapping(format!(
+ "snapshot repository location [{repo_path}] must not contain '..' path components"
+ )),
+ ));
+ }
let base_canon = |p: &std::path::Path| p.canonicalize().unwrap_or_else(|_| p.to_path_buf());
let repo_base = base_canon(std::path::Path::new(repo_path));
let mut allowed_bases: Vec<std::path::PathBuf> = vec![base_canon(data_dir)];
@@ -2053,4 +2071,25 @@ mod snapshot_path_security_tests {
let repo = data.path().to_str().unwrap();
assert!(validate_snapshot_path(repo, "..", &snap, data.path(), &[]).is_err());
}
+
+ #[test]
+ fn f_path_02_residual_parent_ref_to_nonexistent_target_is_rejected() {
+ // The residual escape (#73): a location containing `..` whose fully
+ // resolved target does NOT yet exist. canonicalize() fails, the lexical
+ // fallback keeps the `..`, and a component-based starts_with(data_dir)
+ // would pass — letting create_dir_all write outside data_dir. It must be
+ // refused before any directory is created.
+ let data = tempfile::TempDir::new().unwrap();
+ let repo = format!("{}/../../xerj-escape", data.path().display());
+ let snap = std::path::Path::new(&repo).join("s1");
+ let err = validate_snapshot_path(&repo, "s1", &snap, data.path(), &[])
+ .expect_err("a location containing `..` must be rejected");
+ assert!(err.to_string().contains(".."), "unexpected error: {err}");
+ // And it stays rejected even if the operator allowlists a broad root:
+ // the `..` guard runs before containment, so a parent-ref cannot be
+ // laundered through the allowlist either.
+ let err2 = validate_snapshot_path(&repo, "s1", &snap, data.path(), &["/".to_string()])
+ .expect_err("`..` must be rejected even with a permissive allowlist");
+ assert!(err2.to_string().contains(".."), "unexpected error: {err2}");
+ }
}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index a3e8d78cd..fdc48e8f2 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -1251,6 +1251,44 @@ mod semantic_deadline_regression_tests {
);
}
+ #[tokio::test]
+ async fn explicit_add_fields_enforces_max_fields_per_index() {
+ // #76 S5-5: the explicit mapping path (`add_fields` / PUT _mapping /
+ // schema evolve) must enforce `max_fields_per_index` too. The dynamic
+ // ingest path did, but `Index::add_fields` previously bypassed the cap,
+ // so an authenticated caller could bloat the schema unbounded by sending
+ // fields explicitly.
+ let dir = TempDir::new().unwrap();
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.limits.max_fields_per_index = 50;
+ let engine_instance = Engine::new(config).expect("engine");
+ engine_instance
+ .create_index("field-cap", Schema::empty())
+ .unwrap();
+ let index = engine_instance.get_index("field-cap").unwrap();
+
+ // Fill exactly to the cap (robust to whatever base fields exist).
+ let base = index.schema().await.field_count() as u32;
+ let fill: Vec<FieldConfig> = (0..(50 - base))
+ .map(|i| FieldConfig::new(format!("f{i}"), FieldType::Keyword))
+ .collect();
+ index
+ .add_fields(fill)
+ .await
+ .expect("filling exactly to the cap must succeed");
+
+ // One field over the cap via the explicit path must be REFUSED.
+ let err = index
+ .add_fields(vec![FieldConfig::new("overflow", FieldType::Keyword)])
+ .await
+ .expect_err("exceeding max_fields_per_index on the explicit path must be rejected");
+ assert!(
+ err.to_string().contains("Limit of total fields"),
+ "unexpected error: {err}"
+ );
+ }
+
#[tokio::test]
async fn mapping_vector_over_existing_hidden_chunks_fails_closed_before_visibility() {
let dir = TempDir::new().unwrap();
@@ -8526,18 +8564,27 @@ impl Index {
// useful cross-document scheduling window without giant padding.
let max_passages_per_window = self.embedding_config.onnx_scheduling_window;
let passage_counts = jobs.iter().map(|job| job.texts.len()).collect::<Vec<_>>();
+ // Window boundaries only — do NOT materialize passage texts here.
+ // Pre-building (and cloning) every window's texts up front held
+ // ~N× the request's semantic text for the entire embedding loop
+ // (#71). Keep `(start, end, passages)` and build each window's texts
+ // lazily, MOVING them into `embed_batch`, so only the in-flight
+ // window(s) are resident at a time.
let mut windows = Vec::new();
let mut start = 0;
while start < jobs.len() {
let (end, passages) =
semantic_embedding_window_end(&passage_counts, start, max_passages_per_window);
- let texts: Vec<String> = jobs[start..end]
- .iter()
- .flat_map(|job| job.texts.iter().cloned())
- .collect();
- windows.push((start, end, passages, texts));
+ windows.push((start, end, passages));
start = end;
}
+ // Collect one window's passage texts on demand (borrows `jobs`).
+ let window_texts = |start: usize, end: usize| -> Vec<String> {
+ jobs[start..end]
+ .iter()
+ .flat_map(|job| job.texts.iter().cloned())
+ .collect()
+ };
// Two sessions are an explicit opt-in. Launch at most two complete
// windows together, drain both, then consume results in input
@@ -8549,7 +8596,8 @@ impl Index {
self.embedding_config.onnx_session_pool_size,
) {
let results = collect_ordinal_buffered_two(windows.len(), |ordinal| {
- embedder.embed_batch(windows[ordinal].3.clone())
+ let (start, end, _passages) = windows[ordinal];
+ embedder.embed_batch(window_texts(start, end))
})
.await;
window_results.extend(
@@ -8559,12 +8607,12 @@ impl Index {
.map(|(window, result)| (window.0, window.1, window.2, result)),
);
} else {
- for (start, end, passages, texts) in &windows {
+ for &(start, end, passages) in &windows {
window_results.push((
- *start,
- *end,
- *passages,
- embedder.embed_batch(texts.clone()).await,
+ start,
+ end,
+ passages,
+ embedder.embed_batch(window_texts(start, end)).await,
));
}
}
@@ -14199,6 +14247,22 @@ impl Index {
for field in fields {
candidate.add_field(field)?;
}
+ // Mapping-explosion guard (#76 S5-5): enforce `max_fields_per_index` on
+ // the EXPLICIT mapping path (`PUT /:index/_mapping`,
+ // `POST /v1/schema/:name/evolve`) too, not only the dynamic-ingest path.
+ // `ManagedSchema::add_field` does not check the cap, so without this an
+ // authenticated caller could bypass the mapping-explosion limit by
+ // supplying fields explicitly. Reject before publishing the candidate.
+ let projected = candidate.schema.field_count() as u32;
+ if projected > self.max_fields_per_index {
+ return Err(EngineError::Common(
+ xerj_common::XerjError::invalid_mapping(format!(
+ "Limit of total fields [{}] has been exceeded: this mapping update would \
+ bring the index to {projected} fields",
+ self.max_fields_per_index
+ )),
+ ));
+ }
if self.live_doc_count() > 0 {
let active_before: HashSet<String> = passage_scored_vector_fields(&schema.schema)
.into_iter()