AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(security): persist API keys across restart + honest role surface (item 6)
xerj-org5e7fd7544671
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex a3c1cb970..987bc0003 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -20616,14 +20616,29 @@ pub async fn security_create_api_key(// ES returns base64("id:api_key") as the `encoded` credential.let encoded = base64_encode(&format!("{key_id}:{api_key}"));+ // Honest surface (item 6): `role_descriptors` are accepted for ES/Kibana+ // wire-compatibility but NOT enforced — every key authenticates as the+ // single superuser. Warn loudly so an operator relying on scoped keys is+ // not silently over-privileged. (Full per-key RBAC is deferred.)+ if payload+ .as_ref()+ .and_then(|b| b.get("role_descriptors"))+ .is_some_and(|v| !v.is_null() && v != &json!({}))+ {+ tracing::warn!(+ key = %name,+ "POST /_security/api_key: role_descriptors are NOT enforced; the \+ minted key has full superuser access. See rbac docs."+ );+ }+// Persist the key so the auth middleware can re-authenticate an inbound// `Authorization: ApiKey <encoded>` header. `encoded` decodes to- // `id:api_key`, so the stored secret is the `api_key` value. In-memory- // only (lost on restart); role_descriptors are accepted but not enforced- // (every key authenticates as the single superuser).+ // `id:api_key`, so the stored secret is the `api_key` value. Persisted+ // across restarts via `persist_api_key` (item 6: `<data_dir>/api_keys.json`).let now_ms = Utc::now().timestamp_millis().max(0) as u64;let expiration_ms = parse_api_key_expiration_ms(expiration.as_str(), now_ms);- state.engine.api_keys.insert(+ state.engine.persist_api_key(key_id.clone(),xerj_engine::engine::ApiKeyRecord {name: name.clone(),diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rsindex cb9996308..5edd4ae35 100644--- a/engine/crates/xerj-api/src/native.rs+++ b/engine/crates/xerj-api/src/native.rs@@ -785,11 +785,24 @@ pub async fn audit_verify(State(state): State<AppState>) -> impl IntoResponse {}// ─────────────────────────────────────────────────────────────────────────────-// v0.9 9-P2 — Role store admin+// v0.9 9-P2 — Role store admin — data model only, NOT ENFORCED (RC4 item 6)// ─────────────────────────────────────────────────────────────────────────────+/// Honest-surface banner stamped on every `/_security/role*` response. Roles+/// are stored and round-trip, but the auth path never consults them — every+/// authenticated caller is superuser. See `xerj_engine::rbac` module docs.+const RBAC_NOT_ENFORCED_WARNING: &str =+ "roles are stored but NOT enforced: every authenticated caller has full \+ superuser access regardless of any role assignment. Full RBAC enforcement \+ is deferred.";+pub async fn rbac_list_roles(State(state): State<AppState>) -> impl IntoResponse {- Json(serde_json::json!({ "roles": state.engine.roles.list() })).into_response()+ Json(serde_json::json!({+ "roles": state.engine.roles.list(),+ "enforced": false,+ "warning": RBAC_NOT_ENFORCED_WARNING,+ }))+ .into_response()}pub async fn rbac_get_role(@@ -797,7 +810,12 @@ pub async fn rbac_get_role(axum::extract::Path(name): axum::extract::Path<String>,) -> impl IntoResponse {match state.engine.roles.get(&name) {- Some(r) => Json(r).into_response(),+ Some(r) => Json(serde_json::json!({+ "role": r,+ "enforced": false,+ "warning": RBAC_NOT_ENFORCED_WARNING,+ }))+ .into_response(),None => (axum::http::StatusCode::NOT_FOUND,format!("role '{name}' not found"),@@ -815,7 +833,18 @@ pub async fn rbac_put_role(// Caller may omit the name field; the URL is authoritative.r.name = name;state.engine.roles.put(r.clone());- Json(r).into_response()+ // Loud signal that this PUT grants/restricts nothing (item 6).+ tracing::warn!(+ role = %r.name,+ "PUT /_security/role: role stored but NOT enforced — caller remains superuser"+ );+ Json(serde_json::json!({+ "role": r,+ "created": true,+ "enforced": false,+ "warning": RBAC_NOT_ENFORCED_WARNING,+ }))+ .into_response()}pub async fn rbac_delete_role(@@ -823,7 +852,12 @@ pub async fn rbac_delete_role(axum::extract::Path(name): axum::extract::Path<String>,) -> impl IntoResponse {match state.engine.roles.delete(&name) {- Some(_) => (axum::http::StatusCode::OK, "deleted").into_response(),+ Some(_) => Json(serde_json::json!({+ "deleted": true,+ "enforced": false,+ "warning": RBAC_NOT_ENFORCED_WARNING,+ }))+ .into_response(),None => (axum::http::StatusCode::NOT_FOUND,format!("role '{name}' not found"),diff --git a/engine/crates/xerj-engine/src/engine.rs b/engine/crates/xerj-engine/src/engine.rsindex 8da25acbb..bcd737906 100644--- a/engine/crates/xerj-engine/src/engine.rs+++ b/engine/crates/xerj-engine/src/engine.rs@@ -151,9 +151,15 @@ pub struct DataStream {}/// A created API key, kept in memory so the key can be re-authenticated by-/// the auth middleware. Lost on restart (no persistence yet) and not yet-/// revocable via the DELETE/query endpoints — those are follow-ups.-#[derive(Debug, Clone)]+/// the auth middleware.+///+/// Persisted across restarts (item 6): every mutation rewrites+/// `<data_dir>/api_keys.json` (0600, atomic) and the file is reloaded on boot,+/// so a key minted by `POST /_security/api_key` keeps working after the node+/// restarts — before this, the map was in-memory only and every restart+/// silently invalidated all minted keys (Kibana/agents would 401 until re-set).+/// Serialized as JSON, so the fields must stay `serde`-round-trippable.+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct ApiKeyRecord {/// Caller-supplied key name (informational).pub name: String,@@ -169,6 +175,41 @@ pub struct ApiKeyRecord {pub invalidated: bool,}+/// Atomically write a **secret** file (API-key store) with owner-only (0600)+/// permissions. The temp file is created 0600 *before* any bytes are written so+/// the secret is never briefly world-readable, then renamed over the target+/// (the rename carries the 0600 inode). On non-unix, perms are left to the OS+/// default. Mirrors `index::write_file_atomic` but hardens the mode.+fn write_secret_file_atomic(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> {+ use std::io::Write as _;+ let tmp = path.with_extension("tmp");+ {+ let mut opts = std::fs::OpenOptions::new();+ opts.write(true).create(true).truncate(true);+ #[cfg(unix)]+ {+ use std::os::unix::fs::OpenOptionsExt as _;+ opts.mode(0o600);+ }+ let mut f = opts.open(&tmp)?;+ f.write_all(bytes)?;+ f.sync_all()?;+ }+ // Tighten an already-existing tmp inode too (create() reuses perms).+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt as _;+ std::fs::set_permissions(&tmp, std::fs::Permissions::from_mode(0o600))?;+ }+ std::fs::rename(&tmp, path)?;+ if let Some(parent) = path.parent() {+ if let Ok(dir) = std::fs::File::open(parent) {+ let _ = dir.sync_all();+ }+ }+ Ok(())+}+// ── Engine ────────────────────────────────────────────────────────────────────/// Top-level engine — manages multiple named indices.@@ -394,6 +435,11 @@ impl Engine {}}+ // Restore persisted API keys (item 6) so keys minted before a restart+ // still authenticate. Must run before the server starts accepting+ // requests (i.e. here in `new`), and is cheap (one small JSON file).+ engine.load_persisted_api_keys();+// Spawn the PIT sweeper. Pre-v0.6.2 PITs accumulated forever;// every open without close was a memory leak. The sweeper// walks `engine.pits` every `pit.sweep_interval_secs` and@@ -743,6 +789,72 @@ impl Engine {}}+ /// Path of the persisted API-key store (`<data_dir>/api_keys.json`).+ fn api_keys_path(&self) -> PathBuf {+ self.data_dir.join("api_keys.json")+ }++ /// Insert (or overwrite) an API key and durably persist the whole store+ /// (item 6). The auth middleware re-authenticates `Authorization: ApiKey+ /// <encoded>` against `api_keys`; before this, that map lived only in+ /// memory, so a restart silently invalidated every minted key. Now each+ /// mutation snapshots the full map to `<data_dir>/api_keys.json` (0600,+ /// atomic temp+rename), reloaded on boot.+ ///+ /// The write is best-effort: a persistence failure is logged but does not+ /// fail the create — the key still works until the next restart, matching+ /// the old behavior rather than regressing key creation.+ pub fn persist_api_key(&self, id: String, record: ApiKeyRecord) {+ self.api_keys.insert(id, record);+ self.flush_api_keys();+ }++ /// Serialize the current `api_keys` map to `<data_dir>/api_keys.json`+ /// atomically with owner-only (0600) permissions — the file holds key+ /// secrets, so it must never be group/world readable.+ fn flush_api_keys(&self) {+ let snapshot: std::collections::HashMap<String, ApiKeyRecord> = self+ .api_keys+ .iter()+ .map(|e| (e.key().clone(), e.value().clone()))+ .collect();+ let bytes = match serde_json::to_vec_pretty(&snapshot) {+ Ok(b) => b,+ Err(e) => {+ warn!(error = %e, "failed to serialize api_keys for persistence");+ return;+ }+ };+ if let Err(e) = write_secret_file_atomic(&self.api_keys_path(), &bytes) {+ warn!(error = %e, "failed to persist api_keys.json (keys work until restart)");+ }+ }++ /// Load persisted API keys from `<data_dir>/api_keys.json` into the+ /// in-memory map on boot. A missing file is normal (fresh node / no keys+ /// ever minted); a corrupt file is logged and ignored (the node still+ /// boots — the admin key path is unaffected).+ fn load_persisted_api_keys(&self) {+ let path = self.api_keys_path();+ let Ok(bytes) = std::fs::read(&path) else {+ return;+ };+ match serde_json::from_slice::<std::collections::HashMap<String, ApiKeyRecord>>(&bytes) {+ Ok(map) => {+ let n = map.len();+ for (id, rec) in map {+ self.api_keys.insert(id, rec);+ }+ if n > 0 {+ info!(count = n, "restored persisted API keys");+ }+ }+ Err(e) => {+ warn!(error = %e, "ignoring corrupt api_keys.json");+ }+ }+ }+/// Find the highest-priority template matching `index_name`.fn best_matching_template(&self, index_name: &str) -> Option<IndexTemplate> {let mut best: Option<(i32, IndexTemplate)> = None;diff --git a/engine/crates/xerj-engine/src/rbac.rs b/engine/crates/xerj-engine/src/rbac.rsindex 8a3b17e8f..76eaa2f20 100644--- a/engine/crates/xerj-engine/src/rbac.rs+++ b/engine/crates/xerj-engine/src/rbac.rs@@ -1,18 +1,22 @@-//! Role-based access control — v0.9 9-P2 (skeleton).+//! Role-based access control — data model only. **NOT ENFORCED.**//!-//! Minimal RBAC primitives that downstream auth middleware can grow-//! into. v0.9.0-alpha.1 ships:+//! ⚠️ HONEST-SURFACE WARNING (RC4 item 6): the types below store roles, but+//! **nothing in the auth path consults them.** `is_authorized`+//! (`xerj-api::auth`) is binary — a request either carries the admin key / a+//! valid minted API key (→ full superuser access) or it is rejected. A role+//! `PUT` via `/_security/role/{name}` is recorded and can be read back, but it+//! grants and restricts **nothing**; every authenticated caller is superuser.+//! The `/_security/role*` handlers therefore stamp every response with+//! `"enforced": false` so an operator cannot mistake this for a working+//! authorization system. Full RBAC enforcement (per-request privilege checks,+//! per-key `role_descriptors`, FLS/DLS) is DEFERRED.//!+//! What ships today://! - `Privilege` enum covering the seven core ops (read / write / admin//! index, snapshot create / restore, security admin, audit read).//! - `Role` — name + privileges + index-pattern allow list.//! - `RoleStore` — in-memory map of roles, default seeded with-//! `admin`, `write`, `read`, `read_only_index`, `snapshot_admin`.-//!-//! v0.9.0-beta.1 will wire these into the auth middleware on every-//! handler and add the `PUT /_security/role/{name}` /-//! `PUT /_security/user/{name}` endpoints. Field-level / document-level-//! security (FLS / DLS) lands in v0.9.0-rc.1.+//! `admin`, `write`, `read`, `read_only_index`, `snapshot_admin`, `auditor`.use parking_lot::RwLock;use serde::{Deserialize, Serialize};