AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(security): authenticate gRPC + exempt health probes from auth (RC4 Stream-E blockers 15,16)
xerj-orgab0f55b22930
diff --git a/engine/crates/xerj-api/src/auth.rs b/engine/crates/xerj-api/src/auth.rsindex ee285ff48..2cd87ae17 100644--- a/engine/crates/xerj-api/src/auth.rs+++ b/engine/crates/xerj-api/src/auth.rs@@ -20,17 +20,29 @@ use crate::state::AppState;// ─────────────────────────────────────────────────────────────────────────────+/// Probe endpoints exempted from authentication.+///+/// `/health/live` and `/health/ready` are the conventional Kubernetes liveness+/// and readiness paths that container `HEALTHCHECK`s (see `engine/Dockerfile`)+/// and Helm chart probes hit **without** credentials. Gating them behind auth+/// makes every hardened deployment crashloop: the moment `auth.enabled = true`,+/// the kubelet's probe (and Docker's `curl -fsS .../health/ready`) receives a+/// 401, is marked unhealthy, and the pod is restarted forever. Both handlers+/// are dependency-free and expose no index data (they answer a bare+/// `"live"`/`"ready"` string), so leaving them open is safe.+pub const AUTH_EXEMPT_PATHS: [&str; 2] = ["/health/live", "/health/ready"];+/// Axum middleware that enforces API key authentication.////// Call via `middleware::from_fn_with_state(state, auth_middleware)` in the/// router builders.pub async fn auth_middleware(State(state): State<AppState>, req: Request, next: Next) -> Response {- let cfg = &state.config.auth;-- // Skip auth when disabled or no admin key is configured.- if !cfg.enabled || cfg.admin_api_key.is_empty() {+ // Kubernetes / Docker probes must stay reachable without credentials, or a+ // hardened (auth-enabled) deployment crashloops. See `AUTH_EXEMPT_PATHS`.+ if AUTH_EXEMPT_PATHS.contains(&req.uri().path()) {return next.run(req).await;}+// Xerj Console lives in a peer router (mounted by `xerj-server` via// `Router::merge`) and runs its own session-cookie auth, so this// middleware never sees `/_xerj-console/*` requests — no path-prefix@@ -42,19 +54,41 @@ pub async fn auth_middleware(State(state): State<AppState>, req: Request, next:.get(axum::http::header::AUTHORIZATION).and_then(|v| v.to_str().ok());- let provided_key = match auth_header {- Some(h) => extract_key(h),- None => None,- };+ if is_authorized(&state, auth_header) {+ next.run(req).await+ } else {+ unauthorized_response()+ }+}++/// The shared authorization decision, enforced identically by the HTTP+/// `auth_middleware` and the gRPC auth interceptor (`xerj-server::grpc`) so both+/// API surfaces require the same credentials.+///+/// Returns `true` when the request may proceed:+/// - auth is disabled, or no admin key is configured (open mode — matches the+/// `--insecure` / first-run posture); or+/// - the `Authorization` value carries the configured admin/superuser key; or+/// - it carries a key minted by `POST /_security/api_key` (presented as+/// `ApiKey <base64(id:api_key)>`) that is valid, not expired, and not+/// invalidated.+///+/// `auth_header` is the raw `Authorization` header / `authorization` metadata+/// value (e.g. `"ApiKey abc"` or `"Bearer abc"`), or `None` when absent.+pub fn is_authorized(state: &AppState, auth_header: Option<&str>) -> bool {+ let cfg = &state.config.auth;++ // Skip auth when disabled or no admin key is configured.+ if !cfg.enabled || cfg.admin_api_key.is_empty() {+ return true;+ }- match provided_key {+ match auth_header.and_then(extract_key) {// Fast path: the configured admin/superuser key.- Some(key) if key == cfg.admin_api_key => next.run(req).await,- // A key minted by `POST /_security/api_key`, presented as- // `Authorization: ApiKey <base64(id:api_key)>`, that is valid, not- // expired, and not invalidated.- Some(key) if authenticate_api_key(&state, key) => next.run(req).await,- _ => unauthorized_response(),+ Some(key) if key == cfg.admin_api_key => true,+ // A key minted by `POST /_security/api_key`.+ Some(key) if authenticate_api_key(state, key) => true,+ _ => false,}}@@ -293,4 +327,44 @@ mod tests {"invalidated key should 401");}++ /// Regression for the RC4 blocker: `/health/live` + `/health/ready` must+ /// stay reachable without credentials even when auth is enabled, so Docker+ /// HEALTHCHECKs and k8s probes don't crashloop on a hardened deployment.+ /// Every other route must still demand a key.+ #[tokio::test]+ async fn health_probes_bypass_auth_when_enabled() {+ let admin = "admin-secret-key";+ let state = test_state(admin);+ let app = Router::new()+ .route("/health/live", get(|| async { "live" }))+ .route("/health/ready", get(|| async { "ready" }))+ .route("/_cluster/health", get(|| async { "data" }))+ .layer(middleware::from_fn_with_state(+ state.clone(),+ auth_middleware,+ ))+ .with_state(state);++ // Probes answer 200 with NO credentials, even though auth is on.+ let (status, _) = send(&app, "GET", "/health/live", None, "").await;+ assert_eq!(status, StatusCode::OK, "liveness must bypass auth");+ let (status, _) = send(&app, "GET", "/health/ready", None, "").await;+ assert_eq!(status, StatusCode::OK, "readiness must bypass auth");++ // A normal endpoint still requires the key.+ let (status, _) = send(&app, "GET", "/_cluster/health", None, "").await;+ assert_eq!(+ status,+ StatusCode::UNAUTHORIZED,+ "data endpoint must still 401 without a key"+ );+ let admin_hdr = format!("ApiKey {admin}");+ let (status, _) = send(&app, "GET", "/_cluster/health", Some(&admin_hdr), "").await;+ assert_eq!(+ status,+ StatusCode::OK,+ "data endpoint must pass with the admin key"+ );+ }}diff --git a/engine/crates/xerj-server/src/grpc.rs b/engine/crates/xerj-server/src/grpc.rsindex 093714d76..fe36e247d 100644--- a/engine/crates/xerj-server/src/grpc.rs+++ b/engine/crates/xerj-server/src/grpc.rs@@ -260,15 +260,62 @@ impl XerjSearch for GrpcService {}}+/// gRPC authentication interceptor.+///+/// Runs before every RPC (unary and streaming) and enforces the **same**+/// API-key credentials as the REST / ES-compat `auth_middleware`, via the+/// shared [`xerj_api::auth::is_authorized`] decision. Without it the gRPC+/// listener was fully unauthenticated even with `auth.enabled = true`: any+/// client on the network could read, write, and delete documents (the listener+/// binds `server.bind_address`, default `0.0.0.0`).+///+/// The credential is read from the `authorization` request metadata+/// (`ApiKey <key>` or `Bearer <key>`), mirroring the HTTP `Authorization`+/// header. When auth is disabled (or no admin key is configured — e.g.+/// `--insecure` / first run) the interceptor is a no-op, matching the HTTP+/// surface exactly.+///+/// Note: the tonic listener speaks plaintext h2c (TLS terminates at a reverse+/// proxy or the in-process TLS on the REST/ES listeners), so in an untrusted+/// network the port should still sit behind TLS termination — but it is no+/// longer an unauthenticated open door.+#[derive(Clone)]+struct GrpcAuth {+ state: AppState,+}++impl tonic::service::Interceptor for GrpcAuth {+ fn call(&mut self, request: Request<()>) -> Result<Request<()>, Status> {+ let auth_header = request+ .metadata()+ .get("authorization")+ .and_then(|v| v.to_str().ok());++ if xerj_api::auth::is_authorized(&self.state, auth_header) {+ Ok(request)+ } else {+ Err(Status::unauthenticated(+ "missing or invalid API key in authorization metadata",+ ))+ }+ }+}+/// Serve the `XerjSearch` gRPC service on `addr` until `shutdown` resolves.////// Returns `Err` if the port cannot be bound or the transport fails; callers/// log-and-continue so a gRPC bind failure never takes the whole server down.+///+/// Every RPC is guarded by [`GrpcAuth`], which enforces the same API-key auth+/// as the REST / ES-compat listeners.pub async fn serve_grpc<F>(addr: SocketAddr, state: AppState, shutdown: F) -> anyhow::Result<()>whereF: std::future::Future<Output = ()> + Send + 'static,{- let svc = XerjSearchServer::new(GrpcService::new(state));+ let interceptor = GrpcAuth {+ state: state.clone(),+ };+ let svc = XerjSearchServer::with_interceptor(GrpcService::new(state), interceptor);info!("gRPC XerjSearch listening on {addr}");tonic::transport::Server::builder().add_service(svc)@@ -295,6 +342,17 @@ mod tests {AppState::new(config, engine, metrics)}+ /// Like `app_state`, but with API-key auth enabled and a fixed admin key.+ fn app_state_with_auth(dir: &TempDir, admin_key: &str) -> AppState {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ config.auth.enabled = true;+ config.auth.admin_api_key = admin_key.to_string();+ let metrics = Metrics::new().expect("metrics init");+ let engine = Engine::new(config.clone()).expect("engine init");+ AppState::new(config, engine, metrics)+ }+/// Grab an ephemeral port by binding then dropping a std listener.fn free_port() -> u16 {std::net::TcpListener::bind("127.0.0.1:0")@@ -408,4 +466,122 @@ mod tests {let _ = tx.send(());let _ = server.await;}++ /// Regression for the RC4 security blocker: with `auth.enabled = true` the+ /// gRPC listener was fully unauthenticated — any client could read, write,+ /// and delete. Every RPC must now demand a valid API key (via the+ /// `authorization` metadata), and a correct key must still work end-to-end.+ #[tokio::test]+ async fn grpc_enforces_auth_when_enabled() {+ use tonic::Code;++ let dir = TempDir::new().unwrap();+ let admin = "grpc-admin-secret";+ let state = app_state_with_auth(&dir, admin);++ let port = free_port();+ let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();+ let (tx, rx) = tokio::sync::oneshot::channel::<()>();+ let server = tokio::spawn(serve_grpc(addr, state, async move {+ let _ = rx.await;+ }));++ let url = format!("http://127.0.0.1:{port}");+ let mut client = connect(&url).await;++ // The admin key as an `authorization` metadata value.+ let auth = format!("ApiKey {admin}");+ let key_val: tonic::metadata::MetadataValue<tonic::metadata::Ascii> = auth.parse().unwrap();++ // ── Unauthenticated calls are rejected across read/write/delete ──────+ let e = client+ .health(pb::HealthRequest {})+ .await+ .expect_err("unauth health must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "health: {e:?}");++ let e = client+ .search(pb::SearchRequest {+ index: "grpc_auth".into(),+ query_json: String::new(),+ size: 0,+ from: 0,+ })+ .await+ .expect_err("unauth search must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "search: {e:?}");++ let e = client+ .index(pb::IndexRequest {+ index: "grpc_auth".into(),+ id: "x".into(),+ source_json: r#"{"a":1}"#.into(),+ })+ .await+ .expect_err("unauth index(write) must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "index: {e:?}");++ let e = client+ .get_document(pb::GetRequest {+ index: "grpc_auth".into(),+ id: "x".into(),+ })+ .await+ .expect_err("unauth get must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "get: {e:?}");++ let e = client+ .delete_document(pb::DeleteRequest {+ index: "grpc_auth".into(),+ id: "x".into(),+ })+ .await+ .expect_err("unauth delete must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "delete: {e:?}");++ // ── A wrong key is rejected ─────────────────────────────────────────+ let mut bad = tonic::Request::new(pb::HealthRequest {});+ bad.metadata_mut()+ .insert("authorization", "ApiKey wrong-key".parse().unwrap());+ let e = client+ .health(bad)+ .await+ .expect_err("wrong key must be rejected");+ assert_eq!(e.code(), Code::Unauthenticated, "wrong-key: {e:?}");++ // ── The correct admin key works end-to-end (write → read → delete) ──+ let mut req = tonic::Request::new(pb::IndexRequest {+ index: "grpc_auth".into(),+ id: "x".into(),+ source_json: r#"{"a":1}"#.into(),+ });+ req.metadata_mut().insert("authorization", key_val.clone());+ let indexed = client.index(req).await.expect("authed index").into_inner();+ assert_eq!(indexed.result, "created");++ let mut req = tonic::Request::new(pb::GetRequest {+ index: "grpc_auth".into(),+ id: "x".into(),+ });+ req.metadata_mut().insert("authorization", key_val.clone());+ let got = client+ .get_document(req)+ .await+ .expect("authed get")+ .into_inner();+ assert!(got.found, "authed get should find the doc");++ let mut req = tonic::Request::new(pb::HealthRequest {});+ req.metadata_mut().insert("authorization", key_val.clone());+ let h = client+ .health(req)+ .await+ .expect("authed health")+ .into_inner();+ assert!(!h.status.is_empty(), "authed health status must be set");++ // Shut the server down cleanly.+ let _ = tx.send(());+ let _ = server.await;+ }}