Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(security): authenticate gRPC + exempt health probes from auth (RC4 Stream-E blockers 15,16)

xerj-orgab0f55b22930
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rs
index ee285ff48..2cd87ae17 100644
--- a/engine/crates/xerj-api/src/auth.rs
+++ b/engine/crates/xerj-api/src/auth.rs
@@ -20,17 +20,29 @@ use crate::state::AppState;
// ─────────────────────────────────────────────────────────────────────────────
+/// Probe endpoints exempted from authentication.
+///
+/// `/health/live` and `/health/ready` are the conventional Kubernetes liveness
+/// and readiness paths that container `HEALTHCHECK`s (see `engine/Dockerfile`)
+/// and Helm chart probes hit **without** credentials. Gating them behind auth
+/// makes every hardened deployment crashloop: the moment `auth.enabled = true`,
+/// the kubelet's probe (and Docker's `curl -fsS .../health/ready`) receives a
+/// 401, is marked unhealthy, and the pod is restarted forever. Both handlers
+/// are dependency-free and expose no index data (they answer a bare
+/// `"live"`/`"ready"` string), so leaving them open is safe.
+pub const AUTH_EXEMPT_PATHS: [&str; 2] = ["/health/live", "/health/ready"];
+
/// Axum middleware that enforces API key authentication.
///
/// Call via `middleware::from_fn_with_state(state, auth_middleware)` in the
/// router builders.
pub async fn auth_middleware(State(state): State<AppState>, req: Request, next: Next) -> Response {
- let cfg = &state.config.auth;
-
- // Skip auth when disabled or no admin key is configured.
- if !cfg.enabled || cfg.admin_api_key.is_empty() {
+ // Kubernetes / Docker probes must stay reachable without credentials, or a
+ // hardened (auth-enabled) deployment crashloops. See `AUTH_EXEMPT_PATHS`.
+ if AUTH_EXEMPT_PATHS.contains(&req.uri().path()) {
return next.run(req).await;
}
+
// Xerj Console lives in a peer router (mounted by `xerj-server` via
// `Router::merge`) and runs its own session-cookie auth, so this
// middleware never sees `/_xerj-console/*` requests — no path-prefix
@@ -42,19 +54,41 @@ pub async fn auth_middleware(State(state): State<AppState>, req: Request, next:
.get(axum::http::header::AUTHORIZATION)
.and_then(|v| v.to_str().ok());
- let provided_key = match auth_header {
- Some(h) => extract_key(h),
- None => None,
- };
+ if is_authorized(&state, auth_header) {
+ next.run(req).await
+ } else {
+ unauthorized_response()
+ }
+}
+
+/// The shared authorization decision, enforced identically by the HTTP
+/// `auth_middleware` and the gRPC auth interceptor (`xerj-server::grpc`) so both
+/// API surfaces require the same credentials.
+///
+/// Returns `true` when the request may proceed:
+/// - auth is disabled, or no admin key is configured (open mode — matches the
+/// `--insecure` / first-run posture); or
+/// - the `Authorization` value carries the configured admin/superuser key; or
+/// - it carries a key minted by `POST /_security/api_key` (presented as
+/// `ApiKey <base64(id:api_key)>`) that is valid, not expired, and not
+/// invalidated.
+///
+/// `auth_header` is the raw `Authorization` header / `authorization` metadata
+/// value (e.g. `"ApiKey abc"` or `"Bearer abc"`), or `None` when absent.
+pub fn is_authorized(state: &AppState, auth_header: Option<&str>) -> bool {
+ let cfg = &state.config.auth;
+
+ // Skip auth when disabled or no admin key is configured.
+ if !cfg.enabled || cfg.admin_api_key.is_empty() {
+ return true;
+ }
- match provided_key {
+ match auth_header.and_then(extract_key) {
// Fast path: the configured admin/superuser key.
- Some(key) if key == cfg.admin_api_key => next.run(req).await,
- // A key minted by `POST /_security/api_key`, presented as
- // `Authorization: ApiKey <base64(id:api_key)>`, that is valid, not
- // expired, and not invalidated.
- Some(key) if authenticate_api_key(&state, key) => next.run(req).await,
- _ => unauthorized_response(),
+ Some(key) if key == cfg.admin_api_key => true,
+ // A key minted by `POST /_security/api_key`.
+ Some(key) if authenticate_api_key(state, key) => true,
+ _ => false,
}
}
@@ -293,4 +327,44 @@ mod tests {
"invalidated key should 401"
);
}
+
+ /// Regression for the RC4 blocker: `/health/live` + `/health/ready` must
+ /// stay reachable without credentials even when auth is enabled, so Docker
+ /// HEALTHCHECKs and k8s probes don't crashloop on a hardened deployment.
+ /// Every other route must still demand a key.
+ #[tokio::test]
+ async fn health_probes_bypass_auth_when_enabled() {
+ let admin = "admin-secret-key";
+ let state = test_state(admin);
+ let app = Router::new()
+ .route("/health/live", get(|| async { "live" }))
+ .route("/health/ready", get(|| async { "ready" }))
+ .route("/_cluster/health", get(|| async { "data" }))
+ .layer(middleware::from_fn_with_state(
+ state.clone(),
+ auth_middleware,
+ ))
+ .with_state(state);
+
+ // Probes answer 200 with NO credentials, even though auth is on.
+ let (status, _) = send(&app, "GET", "/health/live", None, "").await;
+ assert_eq!(status, StatusCode::OK, "liveness must bypass auth");
+ let (status, _) = send(&app, "GET", "/health/ready", None, "").await;
+ assert_eq!(status, StatusCode::OK, "readiness must bypass auth");
+
+ // A normal endpoint still requires the key.
+ let (status, _) = send(&app, "GET", "/_cluster/health", None, "").await;
+ assert_eq!(
+ status,
+ StatusCode::UNAUTHORIZED,
+ "data endpoint must still 401 without a key"
+ );
+ let admin_hdr = format!("ApiKey {admin}");
+ let (status, _) = send(&app, "GET", "/_cluster/health", Some(&admin_hdr), "").await;
+ assert_eq!(
+ status,
+ StatusCode::OK,
+ "data endpoint must pass with the admin key"
+ );
+ }
}
diff --git a/engine/crates/xerj-server/src/grpc.rs b/engine/crates/xerj-server/src/grpc.rs
index 093714d76..fe36e247d 100644
--- a/engine/crates/xerj-server/src/grpc.rs
+++ b/engine/crates/xerj-server/src/grpc.rs
@@ -260,15 +260,62 @@ impl XerjSearch for GrpcService {
}
}
+/// gRPC authentication interceptor.
+///
+/// Runs before every RPC (unary and streaming) and enforces the **same**
+/// API-key credentials as the REST / ES-compat `auth_middleware`, via the
+/// shared [`xerj_api::auth::is_authorized`] decision. Without it the gRPC
+/// listener was fully unauthenticated even with `auth.enabled = true`: any
+/// client on the network could read, write, and delete documents (the listener
+/// binds `server.bind_address`, default `0.0.0.0`).
+///
+/// The credential is read from the `authorization` request metadata
+/// (`ApiKey <key>` or `Bearer <key>`), mirroring the HTTP `Authorization`
+/// header. When auth is disabled (or no admin key is configured — e.g.
+/// `--insecure` / first run) the interceptor is a no-op, matching the HTTP
+/// surface exactly.
+///
+/// Note: the tonic listener speaks plaintext h2c (TLS terminates at a reverse
+/// proxy or the in-process TLS on the REST/ES listeners), so in an untrusted
+/// network the port should still sit behind TLS termination — but it is no
+/// longer an unauthenticated open door.
+#[derive(Clone)]
+struct GrpcAuth {
+ state: AppState,
+}
+
+impl tonic::service::Interceptor for GrpcAuth {
+ fn call(&mut self, request: Request<()>) -> Result<Request<()>, Status> {
+ let auth_header = request
+ .metadata()
+ .get("authorization")
+ .and_then(|v| v.to_str().ok());
+
+ if xerj_api::auth::is_authorized(&self.state, auth_header) {
+ Ok(request)
+ } else {
+ Err(Status::unauthenticated(
+ "missing or invalid API key in authorization metadata",
+ ))
+ }
+ }
+}
+
/// Serve the `XerjSearch` gRPC service on `addr` until `shutdown` resolves.
///
/// Returns `Err` if the port cannot be bound or the transport fails; callers
/// log-and-continue so a gRPC bind failure never takes the whole server down.
+///
+/// Every RPC is guarded by [`GrpcAuth`], which enforces the same API-key auth
+/// as the REST / ES-compat listeners.
pub async fn serve_grpc<F>(addr: SocketAddr, state: AppState, shutdown: F) -> anyhow::Result<()>
where
F: std::future::Future<Output = ()> + Send + 'static,
{
- let svc = XerjSearchServer::new(GrpcService::new(state));
+ let interceptor = GrpcAuth {
+ state: state.clone(),
+ };
+ let svc = XerjSearchServer::with_interceptor(GrpcService::new(state), interceptor);
info!("gRPC XerjSearch listening on {addr}");
tonic::transport::Server::builder()
.add_service(svc)
@@ -295,6 +342,17 @@ mod tests {
AppState::new(config, engine, metrics)
}
+ /// Like `app_state`, but with API-key auth enabled and a fixed admin key.
+ fn app_state_with_auth(dir: &TempDir, admin_key: &str) -> AppState {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ config.auth.enabled = true;
+ config.auth.admin_api_key = admin_key.to_string();
+ let metrics = Metrics::new().expect("metrics init");
+ let engine = Engine::new(config.clone()).expect("engine init");
+ AppState::new(config, engine, metrics)
+ }
+
/// Grab an ephemeral port by binding then dropping a std listener.
fn free_port() -> u16 {
std::net::TcpListener::bind("127.0.0.1:0")
@@ -408,4 +466,122 @@ mod tests {
let _ = tx.send(());
let _ = server.await;
}
+
+ /// Regression for the RC4 security blocker: with `auth.enabled = true` the
+ /// gRPC listener was fully unauthenticated — any client could read, write,
+ /// and delete. Every RPC must now demand a valid API key (via the
+ /// `authorization` metadata), and a correct key must still work end-to-end.
+ #[tokio::test]
+ async fn grpc_enforces_auth_when_enabled() {
+ use tonic::Code;
+
+ let dir = TempDir::new().unwrap();
+ let admin = "grpc-admin-secret";
+ let state = app_state_with_auth(&dir, admin);
+
+ let port = free_port();
+ let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
+ let (tx, rx) = tokio::sync::oneshot::channel::<()>();
+ let server = tokio::spawn(serve_grpc(addr, state, async move {
+ let _ = rx.await;
+ }));
+
+ let url = format!("http://127.0.0.1:{port}");
+ let mut client = connect(&url).await;
+
+ // The admin key as an `authorization` metadata value.
+ let auth = format!("ApiKey {admin}");
+ let key_val: tonic::metadata::MetadataValue<tonic::metadata::Ascii> = auth.parse().unwrap();
+
+ // ── Unauthenticated calls are rejected across read/write/delete ──────
+ let e = client
+ .health(pb::HealthRequest {})
+ .await
+ .expect_err("unauth health must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "health: {e:?}");
+
+ let e = client
+ .search(pb::SearchRequest {
+ index: "grpc_auth".into(),
+ query_json: String::new(),
+ size: 0,
+ from: 0,
+ })
+ .await
+ .expect_err("unauth search must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "search: {e:?}");
+
+ let e = client
+ .index(pb::IndexRequest {
+ index: "grpc_auth".into(),
+ id: "x".into(),
+ source_json: r#"{"a":1}"#.into(),
+ })
+ .await
+ .expect_err("unauth index(write) must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "index: {e:?}");
+
+ let e = client
+ .get_document(pb::GetRequest {
+ index: "grpc_auth".into(),
+ id: "x".into(),
+ })
+ .await
+ .expect_err("unauth get must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "get: {e:?}");
+
+ let e = client
+ .delete_document(pb::DeleteRequest {
+ index: "grpc_auth".into(),
+ id: "x".into(),
+ })
+ .await
+ .expect_err("unauth delete must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "delete: {e:?}");
+
+ // ── A wrong key is rejected ─────────────────────────────────────────
+ let mut bad = tonic::Request::new(pb::HealthRequest {});
+ bad.metadata_mut()
+ .insert("authorization", "ApiKey wrong-key".parse().unwrap());
+ let e = client
+ .health(bad)
+ .await
+ .expect_err("wrong key must be rejected");
+ assert_eq!(e.code(), Code::Unauthenticated, "wrong-key: {e:?}");
+
+ // ── The correct admin key works end-to-end (write → read → delete) ──
+ let mut req = tonic::Request::new(pb::IndexRequest {
+ index: "grpc_auth".into(),
+ id: "x".into(),
+ source_json: r#"{"a":1}"#.into(),
+ });
+ req.metadata_mut().insert("authorization", key_val.clone());
+ let indexed = client.index(req).await.expect("authed index").into_inner();
+ assert_eq!(indexed.result, "created");
+
+ let mut req = tonic::Request::new(pb::GetRequest {
+ index: "grpc_auth".into(),
+ id: "x".into(),
+ });
+ req.metadata_mut().insert("authorization", key_val.clone());
+ let got = client
+ .get_document(req)
+ .await
+ .expect("authed get")
+ .into_inner();
+ assert!(got.found, "authed get should find the doc");
+
+ let mut req = tonic::Request::new(pb::HealthRequest {});
+ req.metadata_mut().insert("authorization", key_val.clone());
+ let h = client
+ .health(req)
+ .await
+ .expect("authed health")
+ .into_inner();
+ assert!(!h.status.is_empty(), "authed health status must be set");
+
+ // Shut the server down cleanly.
+ let _ = tx.send(());
+ let _ = server.await;
+ }
}