Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(security): make POST /_security/api_key keys re-authenticatable

xerj-orgbb9a4bf6aab1
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rs
index 11406d9da..ee285ff48 100644
--- a/engine/crates/xerj-api/src/auth.rs
+++ b/engine/crates/xerj-api/src/auth.rs
@@ -48,14 +48,64 @@ pub async fn auth_middleware(State(state): State<AppState>, req: Request, next:
};
match provided_key {
- Some(key) if key == cfg.admin_api_key => {
- // Valid key — allow the request through.
- next.run(req).await
- }
+ // Fast path: the configured admin/superuser key.
+ Some(key) if key == cfg.admin_api_key => next.run(req).await,
+ // A key minted by `POST /_security/api_key`, presented as
+ // `Authorization: ApiKey <base64(id:api_key)>`, that is valid, not
+ // expired, and not invalidated.
+ Some(key) if authenticate_api_key(&state, key) => next.run(req).await,
_ => unauthorized_response(),
}
}
+/// Re-authenticate a created API key credential.
+///
+/// The presented value is `base64("id:api_key")`. Decode it, split on the
+/// first `':'`, look the id up in the engine's key store, and constant-time
+/// compare the secret. All keys authenticate as the single superuser
+/// (role_descriptors are accepted at creation but not enforced).
+fn authenticate_api_key(state: &AppState, presented: &str) -> bool {
+ let decoded = match crate::es_compat::base64_decode(presented) {
+ Some(d) => d,
+ None => return false,
+ };
+ let decoded = match std::str::from_utf8(&decoded) {
+ Ok(s) => s,
+ Err(_) => return false,
+ };
+ let (id, secret) = match decoded.split_once(':') {
+ Some(parts) => parts,
+ None => return false,
+ };
+ let record = match state.engine.api_keys.get(id) {
+ Some(r) => r,
+ None => return false,
+ };
+ if record.invalidated {
+ return false;
+ }
+ if let Some(exp) = record.expiration_ms {
+ let now_ms = chrono::Utc::now().timestamp_millis().max(0) as u64;
+ if now_ms >= exp {
+ return false;
+ }
+ }
+ constant_time_eq(record.secret.as_bytes(), secret.as_bytes())
+}
+
+/// Length-independent-only constant-time byte comparison. Avoids leaking the
+/// secret via early-exit timing once lengths match.
+fn constant_time_eq(a: &[u8], b: &[u8]) -> bool {
+ if a.len() != b.len() {
+ return false;
+ }
+ let mut diff = 0u8;
+ for (x, y) in a.iter().zip(b.iter()) {
+ diff |= x ^ y;
+ }
+ diff == 0
+}
+
// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────
@@ -96,3 +146,151 @@ fn unauthorized_response() -> Response {
(StatusCode::UNAUTHORIZED, Json(body)).into_response()
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Tests
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::{
+ body::Body,
+ http::Request,
+ middleware,
+ routing::{get, post},
+ Router,
+ };
+ use http_body_util::BodyExt;
+ use serde_json::Value;
+ use tower::ServiceExt; // oneshot
+ use xerj_common::{config::Config, metrics::Metrics};
+ use xerj_engine::Engine;
+
+ fn test_state(admin_key: &str) -> AppState {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let path = dir.keep();
+ let mut config = Config::default();
+ config.server.data_dir = path.to_str().unwrap().to_string();
+ config.auth.enabled = true;
+ config.auth.admin_api_key = admin_key.to_string();
+ let metrics = Metrics::new().expect("metrics");
+ let engine = Engine::new(config.clone()).expect("engine");
+ AppState::new(config, engine, metrics)
+ }
+
+ fn app(state: AppState) -> Router {
+ Router::new()
+ .route(
+ "/_security/api_key",
+ post(crate::es_compat::security_create_api_key),
+ )
+ .route(
+ "/_security/_authenticate",
+ get(crate::es_compat::security_authenticate),
+ )
+ .layer(middleware::from_fn_with_state(
+ state.clone(),
+ auth_middleware,
+ ))
+ .with_state(state)
+ }
+
+ async fn send(
+ app: &Router,
+ method: &str,
+ uri: &str,
+ auth: Option<&str>,
+ body: &str,
+ ) -> (StatusCode, Value) {
+ let mut b = Request::builder()
+ .method(method)
+ .uri(uri)
+ .header("content-type", "application/json");
+ if let Some(a) = auth {
+ b = b.header("authorization", a);
+ }
+ let req = b.body(Body::from(body.to_string())).unwrap();
+ let resp = app.clone().oneshot(req).await.unwrap();
+ let status = resp.status();
+ let bytes = resp.into_body().collect().await.unwrap().to_bytes();
+ let json: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ (status, json)
+ }
+
+ #[tokio::test]
+ async fn created_api_key_is_reauthenticatable() {
+ let admin = "admin-secret-key";
+ let state = test_state(admin);
+ let app = app(state.clone());
+ let admin_hdr = format!("ApiKey {admin}");
+
+ // Mint a key (authenticated as admin).
+ let (status, body) = send(
+ &app,
+ "POST",
+ "/_security/api_key",
+ Some(&admin_hdr),
+ r#"{"name":"kibana"}"#,
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "create returned {status}");
+ assert_eq!(body["name"], "kibana");
+ let encoded = body["encoded"].as_str().expect("encoded").to_string();
+ let id = body["id"].as_str().expect("id").to_string();
+
+ // The minted key re-authenticates.
+ let key_hdr = format!("ApiKey {encoded}");
+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;
+ assert_eq!(status, StatusCode::OK, "valid key should authenticate");
+
+ // The admin key still works.
+ let (status, _) = send(
+ &app,
+ "GET",
+ "/_security/_authenticate",
+ Some(&admin_hdr),
+ "",
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK, "admin key should authenticate");
+
+ // A bogus key is rejected.
+ let (status, _) = send(
+ &app,
+ "GET",
+ "/_security/_authenticate",
+ Some("ApiKey bm90LWEta2V5"),
+ "",
+ )
+ .await;
+ assert_eq!(status, StatusCode::UNAUTHORIZED, "bogus key should 401");
+
+ // No auth header at all is rejected.
+ let (status, _) = send(&app, "GET", "/_security/_authenticate", None, "").await;
+ assert_eq!(status, StatusCode::UNAUTHORIZED, "missing auth should 401");
+
+ // Expired key is rejected.
+ state
+ .engine
+ .api_keys
+ .get_mut(&id)
+ .expect("record")
+ .expiration_ms = Some(1); // epoch ms 1 = long past
+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;
+ assert_eq!(status, StatusCode::UNAUTHORIZED, "expired key should 401");
+
+ // Invalidated key is rejected (even after clearing expiry).
+ {
+ let mut rec = state.engine.api_keys.get_mut(&id).expect("record");
+ rec.expiration_ms = None;
+ rec.invalidated = true;
+ }
+ let (status, _) = send(&app, "GET", "/_security/_authenticate", Some(&key_hdr), "").await;
+ assert_eq!(
+ status,
+ StatusCode::UNAUTHORIZED,
+ "invalidated key should 401"
+ );
+ }
+}
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 8fcd80c7f..a566e7549 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -19656,7 +19656,7 @@ pub async fn security_authenticate(State(_state): State<AppState>) -> impl IntoR
// ─────────────────────────────────────────────────────────────────────────────
pub async fn security_create_api_key(
- State(_state): State<AppState>,
+ State(state): State<AppState>,
body: Option<Json<Value>>,
) -> impl IntoResponse {
let payload = body.map(|Json(v)| v);
@@ -19671,12 +19671,31 @@ pub async fn security_create_api_key(
.and_then(|b| b.get("expiration"))
.cloned()
.unwrap_or(Value::Null);
- // Well-formed, unique-per-call key material. Not re-authenticatable.
+ // Well-formed, unique-per-call key material.
let key_id = Uuid::new_v4().to_string();
let raw_secret = format!("{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple());
let api_key = base64_encode(&raw_secret);
// ES returns base64("id:api_key") as the `encoded` credential.
let encoded = base64_encode(&format!("{key_id}:{api_key}"));
+
+ // Persist the key so the auth middleware can re-authenticate an inbound
+ // `Authorization: ApiKey <encoded>` header. `encoded` decodes to
+ // `id:api_key`, so the stored secret is the `api_key` value. In-memory
+ // only (lost on restart); role_descriptors are accepted but not enforced
+ // (every key authenticates as the single superuser).
+ let now_ms = Utc::now().timestamp_millis().max(0) as u64;
+ let expiration_ms = parse_api_key_expiration_ms(expiration.as_str(), now_ms);
+ state.engine.api_keys.insert(
+ key_id.clone(),
+ xerj_engine::engine::ApiKeyRecord {
+ name: name.clone(),
+ secret: api_key.clone(),
+ creation_ms: now_ms,
+ expiration_ms,
+ invalidated: false,
+ },
+ );
+
Json(json!({
"id": key_id,
"name": name,
@@ -19687,6 +19706,63 @@ pub async fn security_create_api_key(
.into_response()
}
+/// Parse an ES time-value expiration (e.g. `"7d"`, `"1h"`, `"30m"`, `"500ms"`)
+/// into an ABSOLUTE expiration in epoch milliseconds relative to `now_ms`.
+/// Returns `None` when the spec is absent or unparseable — meaning the key
+/// never expires.
+fn parse_api_key_expiration_ms(spec: Option<&str>, now_ms: u64) -> Option<u64> {
+ let s = spec?.trim();
+ if s.is_empty() {
+ return None;
+ }
+ // Split the leading number from the trailing unit.
+ let split = s.find(|c: char| c.is_ascii_alphabetic())?;
+ let (num, unit) = s.split_at(split);
+ let n: u64 = num.trim().parse().ok()?;
+ let per_ms: u64 = match unit.trim() {
+ "d" => 86_400_000,
+ "h" => 3_600_000,
+ "m" => 60_000,
+ "s" => 1_000,
+ "ms" => 1,
+ _ => return None,
+ };
+ now_ms.checked_add(n.checked_mul(per_ms)?)
+}
+
+/// Minimal base64 decoder (standard alphabet, tolerates `=` padding). Mirrors
+/// [`base64_encode`]; returns `None` on any invalid input. Used by the auth
+/// middleware to decode an `Authorization: ApiKey <encoded>` credential.
+pub fn base64_decode(input: &str) -> Option<Vec<u8>> {
+ fn val(c: u8) -> Option<u32> {
+ match c {
+ b'A'..=b'Z' => Some((c - b'A') as u32),
+ b'a'..=b'z' => Some((c - b'a' + 26) as u32),
+ b'0'..=b'9' => Some((c - b'0' + 52) as u32),
+ b'+' => Some(62),
+ b'/' => Some(63),
+ _ => None,
+ }
+ }
+ let bytes: Vec<u8> = input.bytes().filter(|&b| b != b'=').collect();
+ let mut out = Vec::with_capacity(bytes.len() * 3 / 4);
+ for chunk in bytes.chunks(4) {
+ let mut acc = 0u32;
+ let mut bits = 0u32;
+ for &b in chunk {
+ acc = (acc << 6) | val(b)?;
+ bits += 6;
+ }
+ // Emit the fully-formed bytes (drop the leftover partial bits from
+ // the dropped padding).
+ while bits >= 8 {
+ bits -= 8;
+ out.push(((acc >> bits) & 0xFF) as u8);
+ }
+ }
+ Some(out)
+}
+
/// Minimal base64 encoder (standard alphabet, no padding variant for ES compat).
fn base64_encode(input: &str) -> String {
let bytes = input.as_bytes();
diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rs
index 54814351d..ad2da0d88 100644
--- a/engine/crates/xerj-engine/src/engine.rs
+++ b/engine/crates/xerj-engine/src/engine.rs
@@ -133,6 +133,25 @@ pub struct DataStream {
pub generation: u64,
}
+/// A created API key, kept in memory so the key can be re-authenticated by
+/// the auth middleware. Lost on restart (no persistence yet) and not yet
+/// revocable via the DELETE/query endpoints — those are follow-ups.
+#[derive(Debug, Clone)]
+pub struct ApiKeyRecord {
+ /// Caller-supplied key name (informational).
+ pub name: String,
+ /// The secret half of the credential — the `api_key` value returned to
+ /// the caller, i.e. the part after `id:` in the decoded `ApiKey` header.
+ pub secret: String,
+ /// Creation time in epoch milliseconds.
+ pub creation_ms: u64,
+ /// Absolute expiration in epoch milliseconds, or `None` if the key never
+ /// expires.
+ pub expiration_ms: Option<u64>,
+ /// Set once the key has been invalidated (revoked).
+ pub invalidated: bool,
+}
+
// ── Engine ────────────────────────────────────────────────────────────────────
/// Top-level engine — manages multiple named indices.
@@ -184,6 +203,10 @@ pub struct Engine {
pub rollup_jobs: Arc<DashMap<String, Value>>,
/// CCR auto-follow pattern name → pattern JSON
pub ccr_auto_follow: Arc<DashMap<String, Value>>,
+ /// API key id → record. Populated by `POST /_security/api_key` so the
+ /// auth middleware can re-authenticate `Authorization: ApiKey <encoded>`.
+ /// In-memory only (lost on restart).
+ pub api_keys: Arc<DashMap<String, ApiKeyRecord>>,
/// legacy index template name (v1 /_template) → template JSON
pub legacy_templates: Arc<DashMap<String, Value>>,
/// pipeline_name → compiled, executable Pipeline (typed transform pipeline)
@@ -273,6 +296,7 @@ impl Engine {
frozen_indices: Arc::new(DashMap::new()),
rollup_jobs: Arc::new(DashMap::new()),
ccr_auto_follow: Arc::new(DashMap::new()),
+ api_keys: Arc::new(DashMap::new()),
legacy_templates: Arc::new(DashMap::new()),
transform_pipelines: Arc::new(DashMap::new()),
pits: Arc::new(DashMap::new()),
diff --git a/engine/tests/es-compat-yaml/src/main.rs b/engine/tests/es-compat-yaml/src/main.rs
index 1ba3a1023..b753b09a3 100644
--- a/engine/tests/es-compat-yaml/src/main.rs
+++ b/engine/tests/es-compat-yaml/src/main.rs
@@ -1355,6 +1355,9 @@ fn resolve_action(action: &str, params: &serde_yaml::Mapping) -> (String, String
("PUT".into(), format!("/_ccr/auto_follow/{}", name), body)
}
+ // Security — API keys
+ "security.create_api_key" => ("POST".into(), "/_security/api_key".into(), body),
+
// Catch-all
_ => {
let path = format!("/{}", action.replace('.', "/"));
diff --git a/engine/tests/es-compat-yaml/yaml/security/10_api_key.yml b/engine/tests/es-compat-yaml/yaml/security/10_api_key.yml
new file mode 100644
index 000000000..9930cbba7
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/security/10_api_key.yml
@@ -0,0 +1,14 @@
+---
+"create API key returns a well-formed, re-authenticatable credential":
+ # The yaml runner boots XERJ --insecure (auth disabled), so the
+ # authenticate round-trip can't be exercised here — that is covered by a
+ # Rust integration test. This asserts the create response shape/values.
+ - do:
+ security.create_api_key:
+ body:
+ name: kibana
+
+ - match: { name: kibana }
+ - is_true: id
+ - is_true: api_key
+ - is_true: encoded