AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(security): make POST /_security/api_key keys re-authenticatable
xerj-orgbb9a4bf6aab1
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rsindex 11406d9da..ee285ff48 100644--- a/engine/crates/xerj-api/src/auth.rs+++ b/engine/crates/xerj-api/src/auth.rs@@ -48,14 +48,64 @@ pub async fn auth_middleware(State(state): State<AppState>, req: Request, next:};match provided_key {- Some(key) if key == cfg.admin_api_key => {- // Valid key — allow the request through.- next.run(req).await- }+ // Fast path: the configured admin/superuser key.+ Some(key) if key == cfg.admin_api_key => next.run(req).await,+ // A key minted by `POST /_security/api_key`, presented as+ // `Authorization: ApiKey <base64(id:api_key)>`, that is valid, not+ // expired, and not invalidated.+ Some(key) if authenticate_api_key(&state, key) => next.run(req).await,_ => unauthorized_response(),}}+/// Re-authenticate a created API key credential.+///+/// The presented value is `base64("id:api_key")`. Decode it, split on the+/// first `':'`, look the id up in the engine's key store, and constant-time+/// compare the secret. All keys authenticate as the single superuser+/// (role_descriptors are accepted at creation but not enforced).+fn authenticate_api_key(state: &AppState, presented: &str) -> bool {+ let decoded = match crate::es_compat::base64_decode(presented) {+ Some(d) => d,+ None => return false,+ };+ let decoded = match std::str::from_utf8(&decoded) {+ Ok(s) => s,+ Err(_) => return false,+ };+ let (id, secret) = match decoded.split_once(':') {+ Some(parts) => parts,+ None => return false,+ };+ let record = match state.engine.api_keys.get(id) {+ Some(r) => r,+ None => return false,+ };+ if record.invalidated {+ return false;+ }+ if let Some(exp) = record.expiration_ms {+ let now_ms = chrono::Utc::now().timestamp_millis().max(0) as u64;+ if now_ms >= exp {+ return false;+ }+ }+ constant_time_eq(record.secret.as_bytes(), secret.as_bytes())+}++/// Length-independent-only constant-time byte comparison. Avoids leaking the+/// secret via early-exit timing once lengths match.+fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {+ if a.len() != b.len() {+ return false;+ }+ let mut diff = 0u8;+ for (x, y) in a.iter().zip(b.iter()) {+ diff |= x ^ y;+ }+ diff == 0+}+// ─────────────────────────────────────────────────────────────────────────────// Helpers// ─────────────────────────────────────────────────────────────────────────────@@ -96,3 +146,151 @@ fn unauthorized_response() -> Response {(StatusCode::UNAUTHORIZED, Json(body)).into_response()}++// ─────────────────────────────────────────────────────────────────────────────+// Tests+// ─────────────────────────────────────────────────────────────────────────────++#[cfg(test)]+mod tests {+ use super::*;+ use axum::{+ body::Body,+ http::Request,+ middleware,+ routing::{get, post},+ Router,+ };+ use http_body_util::BodyExt;+ use serde_json::Value;+ use tower::ServiceExt; // oneshot+ use xerj_common::{config::Config, metrics::Metrics};+ use xerj_engine::Engine;++ fn test_state(admin_key: &str) -> AppState {+ let dir = tempfile::tempdir().expect("tempdir");+ let path = dir.keep();+ let mut config = Config::default();+ config.server.data_dir = path.to_str().unwrap().to_string();+ config.auth.enabled = true;+ config.auth.admin_api_key = admin_key.to_string();+ let metrics = Metrics::new().expect("metrics");+ let engine = Engine::new(config.clone()).expect("engine");+ AppState::new(config, engine, metrics)+ }++ fn app(state: AppState) -> Router {+ Router::new()+ .route(+ "/_security/api_key",+ post(crate::es_compat::security_create_api_key),+ )+ .route(+ "/_security/_authenticate",+ get(crate::es_compat::security_authenticate),+ )+ .layer(middleware::from_fn_with_state(+ state.clone(),+ auth_middleware,+ ))+ .with_state(state)+ }++ async fn send(+ app: &Router,+ method: &str,+ uri: &str,+ auth: Option<&str>,+ body: &str,+ ) -> (StatusCode, Value) {+ let mut b = Request::builder()+ .method(method)+ .uri(uri)+ .header("content-type", "application/json");+ if let Some(a) = auth {+ b = b.header("authorization", a);+ }+ let req = b.body(Body::from(body.to_string())).unwrap();+ let resp = app.clone().oneshot(req).await.unwrap();+ let status = resp.status();+ let bytes = resp.into_body().collect().await.unwrap().to_bytes();+ let json: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ (status, json)+ }++ #[tokio::test]+ async fn created_api_key_is_reauthenticatable() {+ let admin = "admin-secret-key";+ let state = test_state(admin);+ let app = app(state.clone());+ let admin_hdr = format!("ApiKey {admin}");++ // Mint a key (authenticated as admin).+ let (status, body) = send(+ &app,+ "POST",+ "/_security/api_key",+ Some(&admin_hdr),+ r#"{"name":"kibana"}"#,+ )+ .await;+ assert_eq!(status, StatusCode::OK, "create returned {status}");+ assert_eq!(body["name"], "kibana");+ let encoded = body["encoded"].as_str().expect("encoded").to_string();+ let id = body["id"].as_str().expect("id").to_string();++ // The minted key re-authenticates.+ let key_hdr = format!("ApiKey {encoded}");+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;+ assert_eq!(status, StatusCode::OK, "valid key should authenticate");++ // The admin key still works.+ let (status, _) = send(+ &app,+ "GET",+ "/_security/_authenticate",+ Some(&admin_hdr),+ "",+ )+ .await;+ assert_eq!(status, StatusCode::OK, "admin key should authenticate");++ // A bogus key is rejected.+ let (status, _) = send(+ &app,+ "GET",+ "/_security/_authenticate",+ Some("ApiKey bm90LWEta2V5"),+ "",+ )+ .await;+ assert_eq!(status, StatusCode::UNAUTHORIZED, "bogus key should 401");++ // No auth header at all is rejected.+ let (status, _) = send(&app, "GET", "/_security/_authenticate", None, "").await;+ assert_eq!(status, StatusCode::UNAUTHORIZED, "missing auth should 401");++ // Expired key is rejected.+ state+ .engine+ .api_keys+ .get_mut(&id)+ .expect("record")+ .expiration_ms = Some(1); // epoch ms 1 = long past+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;+ assert_eq!(status, StatusCode::UNAUTHORIZED, "expired key should 401");++ // Invalidated key is rejected (even after clearing expiry).+ {+ let mut rec = state.engine.api_keys.get_mut(&id).expect("record");+ rec.expiration_ms = None;+ rec.invalidated = true;+ }+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;+ assert_eq!(+ status,+ StatusCode::UNAUTHORIZED,+ "invalidated key should 401"+ );+ }+}diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex 8fcd80c7f..a566e7549 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -19656,7 +19656,7 @@ pub async fn security_authenticate(State(_state): State<AppState>) -> impl IntoR// ─────────────────────────────────────────────────────────────────────────────pub async fn security_create_api_key(- State(_state): State<AppState>,+ State(state): State<AppState>,body: Option<Json<Value>>,) -> impl IntoResponse {let payload = body.map(|Json(v)| v);@@ -19671,12 +19671,31 @@ pub async fn security_create_api_key(.and_then(|b| b.get("expiration")).cloned().unwrap_or(Value::Null);- // Well-formed, unique-per-call key material. Not re-authenticatable.+ // Well-formed, unique-per-call key material.let key_id = Uuid::new_v4().to_string();let raw_secret = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple());let api_key = base64_encode(&raw_secret);// ES returns base64("id:api_key") as the `encoded` credential.let encoded = base64_encode(&format!("{key_id}:{api_key}"));++ // Persist the key so the auth middleware can re-authenticate an inbound+ // `Authorization: ApiKey <encoded>` header. `encoded` decodes to+ // `id:api_key`, so the stored secret is the `api_key` value. In-memory+ // only (lost on restart); role_descriptors are accepted but not enforced+ // (every key authenticates as the single superuser).+ let now_ms = Utc::now().timestamp_millis().max(0) as u64;+ let expiration_ms = parse_api_key_expiration_ms(expiration.as_str(), now_ms);+ state.engine.api_keys.insert(+ key_id.clone(),+ xerj_engine::engine::ApiKeyRecord {+ name: name.clone(),+ secret: api_key.clone(),+ creation_ms: now_ms,+ expiration_ms,+ invalidated: false,+ },+ );+Json(json!({"id": key_id,"name": name,@@ -19687,6 +19706,63 @@ pub async fn security_create_api_key(.into_response()}+/// Parse an ES time-value expiration (e.g. `"7d"`, `"1h"`, `"30m"`, `"500ms"`)+/// into an ABSOLUTE expiration in epoch milliseconds relative to `now_ms`.+/// Returns `None` when the spec is absent or unparseable — meaning the key+/// never expires.+fn parse_api_key_expiration_ms(spec: Option<&str>, now_ms: u64) -> Option<u64> {+ let s = spec?.trim();+ if s.is_empty() {+ return None;+ }+ // Split the leading number from the trailing unit.+ let split = s.find(|c: char| c.is_ascii_alphabetic())?;+ let (num, unit) = s.split_at(split);+ let n: u64 = num.trim().parse().ok()?;+ let per_ms: u64 = match unit.trim() {+ "d" => 86_400_000,+ "h" => 3_600_000,+ "m" => 60_000,+ "s" => 1_000,+ "ms" => 1,+ _ => return None,+ };+ now_ms.checked_add(n.checked_mul(per_ms)?)+}++/// Minimal base64 decoder (standard alphabet, tolerates `=` padding). Mirrors+/// [`base64_encode`]; returns `None` on any invalid input. Used by the auth+/// middleware to decode an `Authorization: ApiKey <encoded>` credential.+pub fn base64_decode(input: &str) -> Option<Vec<u8>> {+ fn val(c: u8) -> Option<u32> {+ match c {+ b'A'..=b'Z' => Some((c - b'A') as u32),+ b'a'..=b'z' => Some((c - b'a' + 26) as u32),+ b'0'..=b'9' => Some((c - b'0' + 52) as u32),+ b'+' => Some(62),+ b'/' => Some(63),+ _ => None,+ }+ }+ let bytes: Vec<u8> = input.bytes().filter(|&b| b != b'=').collect();+ let mut out = Vec::with_capacity(bytes.len() * 3 / 4);+ for chunk in bytes.chunks(4) {+ let mut acc = 0u32;+ let mut bits = 0u32;+ for &b in chunk {+ acc = (acc << 6) | val(b)?;+ bits += 6;+ }+ // Emit the fully-formed bytes (drop the leftover partial bits from+ // the dropped padding).+ while bits >= 8 {+ bits -= 8;+ out.push(((acc >> bits) & 0xFF) as u8);+ }+ }+ Some(out)+}+/// Minimal base64 encoder (standard alphabet, no padding variant for ES compat).fn base64_encode(input: &str) -> String {let bytes = input.as_bytes();diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 54814351d..ad2da0d88 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -133,6 +133,25 @@ pub struct DataStream {pub generation: u64,}+/// A created API key, kept in memory so the key can be re-authenticated by+/// the auth middleware. Lost on restart (no persistence yet) and not yet+/// revocable via the DELETE/query endpoints — those are follow-ups.+#[derive(Debug, Clone)]+pub struct ApiKeyRecord {+ /// Caller-supplied key name (informational).+ pub name: String,+ /// The secret half of the credential — the `api_key` value returned to+ /// the caller, i.e. the part after `id:` in the decoded `ApiKey` header.+ pub secret: String,+ /// Creation time in epoch milliseconds.+ pub creation_ms: u64,+ /// Absolute expiration in epoch milliseconds, or `None` if the key never+ /// expires.+ pub expiration_ms: Option<u64>,+ /// Set once the key has been invalidated (revoked).+ pub invalidated: bool,+}+// ── Engine ────────────────────────────────────────────────────────────────────/// Top-level engine — manages multiple named indices.@@ -184,6 +203,10 @@ pub struct Engine {pub rollup_jobs: Arc<DashMap<String, Value>>,/// CCR auto-follow pattern name → pattern JSONpub ccr_auto_follow: Arc<DashMap<String, Value>>,+ /// API key id → record. Populated by `POST /_security/api_key` so the+ /// auth middleware can re-authenticate `Authorization: ApiKey <encoded>`.+ /// In-memory only (lost on restart).+ pub api_keys: Arc<DashMap<String, ApiKeyRecord>>,/// legacy index template name (v1 /_template) → template JSONpub legacy_templates: Arc<DashMap<String, Value>>,/// pipeline_name → compiled, executable Pipeline (typed transform pipeline)@@ -273,6 +296,7 @@ impl Engine {frozen_indices: Arc::new(DashMap::new()),rollup_jobs: Arc::new(DashMap::new()),ccr_auto_follow: Arc::new(DashMap::new()),+ api_keys: Arc::new(DashMap::new()),legacy_templates: Arc::new(DashMap::new()),transform_pipelines: Arc::new(DashMap::new()),pits: Arc::new(DashMap::new()),diff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rsindex 1ba3a1023..b753b09a3 100644--- a/engine/tests/es-compat-yaml/src/main.rs+++ b/engine/tests/es-compat-yaml/src/main.rs@@ -1355,6 +1355,9 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String("PUT".into(), format!("/_ccr/auto_follow/{}", name), body)}+ // Security — API keys+ "security.create_api_key" => ("POST".into(), "/_security/api_key".into(), body),+// Catch-all_ => {let path = format!("/{}", action.replace('.', "/"));diff --git a/engine/tests/es-compat-yaml/yaml/security/10_api_key.yml b/engine/tests/es-compat-yaml/yaml/security/10_api_key.ymlnew file mode 100644index 000000000..9930cbba7--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/security/10_api_key.yml@@ -0,0 +1,14 @@+---+"create API key returns a well-formed, re-authenticatable credential":+ # The yaml runner boots XERJ --insecure (auth disabled), so the+ # authenticate round-trip can't be exercised here — that is covered by a+ # Rust integration test. This asserts the create response shape/values.+ - do:+ security.create_api_key:+ body:+ name: kibana++ - match: { name: kibana }+ - is_true: id+ - is_true: api_key+ - is_true: encoded