Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

53.9 KB
Raw Blame
//! Real tonic gRPC server for the `XerjSearch` service (default `:8081`), and
//! the Corpus Hub `XerjHub` service on the same listener behind the same
//! interceptor (`grpc_hub.rs`).
//!
//! Replaces the v0.1 placeholder (which only bound the port and dropped
//! connections). The service is a thin adapter over the same `Engine`/`Index`
//! methods the REST and ES-compat listeners use, so gRPC clients get identical
//! semantics. Wire schema: `proto/xerj.proto` (package `xerj.v1`), compiled to
//! Rust by `build.rs` (pure-Rust `protox`, no host `protoc`).
//!
//! The listener speaks plaintext HTTP/2 (h2c). TLS termination stays with the
//! REST/ES listeners (axum-server + ring); tonic is built without its `tls`
//! feature so no second crypto backend is dragged in.
//!
//! That is a real gap, not a footnote (issue #229): `tls.enabled = true`
//! encrypts the other two listeners and leaves this one in the clear, and
//! nothing a working gRPC client sees would tell the operator. Startup
//! therefore refuses the combination "TLS on + non-loopback bind" unless
//! `tls.allow_insecure_grpc_h2c` declares it intended — see
//! `Config::grpc_h2c_exposed_off_loopback` and step 5b of `main.rs`. Wiring
//! tonic's own `tls` feature (a second crypto backend beside axum-server's
//! ring) is the open follow-up; documented in `docs/SECURITY_MODEL.md`.

// Only the test-only `serve_grpc` takes an address now; the server hands in
// a listener it already bound (issue #465).
#[cfg(test)]
use std::net::SocketAddr;

use anyhow::Context;
use tonic::{Request, Response, Status, Streaming};
use tracing::info;

use xerj_api::auth::Principal;
use xerj_api::AppState;
use xerj_engine::rbac::Privilege;
use xerj_query::parse_request;

/// Generated prost messages + tonic client/server stubs for `xerj.v1`.
///
/// The generated code is exempted from lints — it is machine-emitted and not
/// meant to satisfy this crate's `-D warnings` clippy gate.
#[allow(clippy::all, clippy::pedantic, clippy::nursery, missing_docs)]
pub mod pb {
    tonic::include_proto!("xerj.v1");
}

use pb::xerj_search_server::{XerjSearch, XerjSearchServer};

/// gRPC adapter over the engine. Clone is cheap — `AppState` is all `Arc`s.
#[derive(Clone)]
pub struct GrpcService {
    state: AppState,
}

impl GrpcService {
    /// Build the service from shared server state.
    pub fn new(state: AppState) -> Self {
        Self { state }
    }
}

/// Authorize an RPC that names an index.
///
/// The listener speaks the same engine as the HTTP surfaces, so the reserved
/// `.xerj-memory-*` namespace has to be gated here too (issue #79) — otherwise
/// per-brain authorization would hold on :9200 and :8080 and fall open on
/// :8081, which is worse than not having it. [`GrpcAuth`] plants the
/// [`Principal`] in the request extensions; a missing one means the
/// interceptor did not run, and is treated as no credential at all.
///
/// `clippy::result_large_err` fires because `tonic::Status` is a fat value, but
/// `Err` here IS the RPC's denial status — the shape every other handler in
/// this file already returns — so boxing it would add an allocation on the deny
/// path and force an unwrap at every `?`.
#[allow(clippy::result_large_err)]
fn authorize(
    extensions: &tonic::Extensions,
    index: &str,
    privilege: Privilege,
) -> Result<(), Status> {
    let principal = extensions.get::<Principal>().unwrap_or(&Principal::Denied);
    if principal.allows_index(index, privilege) {
        Ok(())
    } else {
        Err(Status::permission_denied(format!(
            "this credential is not authorized for [{index}]"
        )))
    }
}

/// Same rule as the HTTP middleware: a pattern that could reach the reserved
/// namespace is refused rather than expanded, and a scoped credential must
/// name the index it was granted.
#[allow(clippy::result_large_err)] // same rationale as `authorize`
fn reject_wildcards(principal: &Principal, index: &str) -> Result<(), Status> {
    if (index.contains('*') || index == "_all") && !principal.is_superuser() {
        return Err(Status::permission_denied(format!(
            "index patterns are not available to this credential: [{index}]"
        )));
    }
    Ok(())
}

#[tonic::async_trait]
impl XerjSearch for GrpcService {
    async fn search(
        &self,
        request: Request<pb::SearchRequest>,
    ) -> Result<Response<pb::SearchResponse>, Status> {
        let (_meta, extensions, req) = request.into_parts();
        let principal = extensions.get::<Principal>().unwrap_or(&Principal::Denied);
        reject_wildcards(principal, &req.index)?;
        // Before `get_index`, so 403-vs-not-found cannot map the node's brains.
        authorize(&extensions, &req.index, Privilege::ReadIndex)?;

        let idx = self
            .state
            .engine
            .get_index(&req.index)
            .map_err(|e| Status::not_found(e.to_string()))?;

        // `query_json` is the ES query DSL. We accept both a bare query clause
        // (e.g. `{"match":{...}}`) and a full request body (`{"query":{...},
        // "size":...}`); an empty string means match_all. The dedicated
        // `size`/`from` fields override the body when set (non-zero).
        let mut body: serde_json::Value = if req.query_json.trim().is_empty() {
            serde_json::json!({ "query": { "match_all": {} } })
        } else {
            serde_json::from_str(&req.query_json).map_err(|e| {
                Status::invalid_argument(format!("query_json is not valid JSON: {e}"))
            })?
        };
        // `query_json` may be a full request body, and `parse_request` ignores
        // keys it does not know — so a `rerank` block vanished here and the
        // caller got the engine's order back, believing it had been reranked.
        // Refused by name instead. Checked before the bare-clause wrap below,
        // which would otherwise bury the key inside `query`.
        // `carries_rerank`, the rule every HTTP surface uses: `"rerank": null`
        // is an absent block, not a rerank request. A `judge` block (issue
        // #1060) is refused for the same reason, through the same
        // second-stage check the raw-JSON HTTP surfaces use — naming the
        // block the body actually carries.
        if xerj_api::judge_stage::carries_second_stage(&body) {
            let reason = if xerj_api::rerank_stage::carries_rerank(&body) {
                xerj_api::rerank_stage::unsupported_reason("the gRPC Search RPC")
            } else {
                xerj_api::judge_stage::unsupported_reason("the gRPC Search RPC")
            };
            return Err(Status::invalid_argument(reason));
        }
        if body.get("query").is_none() {
            body = serde_json::json!({ "query": body });
        }
        if req.size != 0 {
            body["size"] = serde_json::json!(req.size);
        }
        if req.from != 0 {
            body["from"] = serde_json::json!(req.from);
        }

        let search_req = parse_request(&body)
            .map_err(|e| Status::invalid_argument(format!("invalid query: {e}")))?;

        let started = std::time::Instant::now();
        let result = idx
            .search(&search_req)
            .await
            .map_err(|e| Status::internal(e.to_string()))?;
        let took_ms = started.elapsed().as_millis() as i64;

        let hits = result
            .hits
            .iter()
            .map(|h| pb::Hit {
                id: h.id.clone(),
                score: h.score,
                source_json: h.source.to_string(),
                index: req.index.clone(),
            })
            .collect();

        Ok(Response::new(pb::SearchResponse {
            total_hits: result.total.value as i64,
            hits,
            took_ms,
            timed_out: result.timed_out,
        }))
    }

    async fn index(
        &self,
        request: Request<pb::IndexRequest>,
    ) -> Result<Response<pb::IndexResponse>, Status> {
        let (_meta, extensions, req) = request.into_parts();
        authorize(&extensions, &req.index, Privilege::WriteIndex)?;

        // Index-on-write: ES auto-creates the index on first document.
        let idx = self
            .state
            .engine
            .get_or_create_index(&req.index)
            .map_err(|e| Status::invalid_argument(e.to_string()))?;

        let source: serde_json::Value = serde_json::from_str(&req.source_json)
            .map_err(|e| Status::invalid_argument(format!("source_json is not valid JSON: {e}")))?;

        let id = if req.id.is_empty() {
            None
        } else {
            Some(req.id.clone())
        };

        let resp = idx
            .index_document(id, source)
            .await
            .map_err(|e| Status::internal(e.to_string()))?;
        self.state.metrics.record_doc_indexed(&req.index);

        Ok(Response::new(pb::IndexResponse {
            id: resp.id,
            version: resp.version as i64,
            result: resp.result,
        }))
    }

    async fn bulk_index(
        &self,
        request: Request<Streaming<pb::IndexRequest>>,
    ) -> Result<Response<pb::BulkResponse>, Status> {
        let (_meta, extensions, mut stream) = request.into_parts();
        let started = std::time::Instant::now();
        let mut indexed = 0i32;
        let mut errors = false;

        while let Some(item) = stream.message().await? {
            // Per message: one stream may name many indices, and an
            // unauthorized one must not ride in on an authorized one. A refused
            // item is counted as an error, matching how this loop already
            // reports a bad index or an unparsable source.
            if authorize(&extensions, &item.index, Privilege::WriteIndex).is_err() {
                errors = true;
                continue;
            }
            let idx = match self.state.engine.get_or_create_index(&item.index) {
                Ok(i) => i,
                Err(_) => {
                    errors = true;
                    continue;
                }
            };
            let source: serde_json::Value = match serde_json::from_str(&item.source_json) {
                Ok(v) => v,
                Err(_) => {
                    errors = true;
                    continue;
                }
            };
            let id = if item.id.is_empty() {
                None
            } else {
                Some(item.id)
            };
            if idx.index_document(id, source).await.is_ok() {
                indexed += 1;
                self.state.metrics.record_doc_indexed(&item.index);
            } else {
                errors = true;
            }
        }

        Ok(Response::new(pb::BulkResponse {
            took_ms: started.elapsed().as_millis() as i64,
            errors,
            indexed,
        }))
    }

    async fn get_document(
        &self,
        request: Request<pb::GetRequest>,
    ) -> Result<Response<pb::GetResponse>, Status> {
        let (_meta, extensions, req) = request.into_parts();
        // Deliberately a hard denial rather than this RPC's soft "not found":
        // answering `found: false` for an unauthorized index would still let a
        // caller probe which indices exist by timing or by error shape.
        authorize(&extensions, &req.index, Privilege::ReadIndex)?;

        // Unknown index → "not found" (ES GET semantics), not an error.
        let idx = match self.state.engine.get_index(&req.index) {
            Ok(i) => i,
            Err(_) => {
                return Ok(Response::new(pb::GetResponse {
                    found: false,
                    id: req.id,
                    source_json: String::new(),
                    version: 0,
                }));
            }
        };

        match idx
            .get_document(&req.id)
            .await
            .map_err(|e| Status::internal(e.to_string()))?
        {
            Some(source) => Ok(Response::new(pb::GetResponse {
                found: true,
                id: req.id,
                source_json: source.to_string(),
                version: 1,
            })),
            None => Ok(Response::new(pb::GetResponse {
                found: false,
                id: req.id,
                source_json: String::new(),
                version: 0,
            })),
        }
    }

    async fn delete_document(
        &self,
        request: Request<pb::DeleteRequest>,
    ) -> Result<Response<pb::DeleteResponse>, Status> {
        let (_meta, extensions, req) = request.into_parts();
        authorize(&extensions, &req.index, Privilege::WriteIndex)?;

        let idx = self
            .state
            .engine
            .get_index(&req.index)
            .map_err(|e| Status::not_found(e.to_string()))?;

        let existed = idx
            .delete_document(&req.id)
            .await
            .map_err(|e| Status::internal(e.to_string()))?;

        Ok(Response::new(pb::DeleteResponse {
            result: if existed { "deleted" } else { "not_found" }.to_string(),
        }))
    }

    async fn health(
        &self,
        _request: Request<pb::HealthRequest>,
    ) -> Result<Response<pb::HealthResponse>, Status> {
        let health = self.state.engine.health().await;
        Ok(Response::new(pb::HealthResponse {
            status: health.status,
            num_indices: health.index_count as i32,
            total_docs: health.total_docs as i64,
        }))
    }
}

/// gRPC authentication interceptor.
///
/// Runs before every RPC (unary and streaming) and enforces the **same**
/// API-key credentials as the REST / ES-compat `auth_middleware`, via the
/// shared [`xerj_api::auth::is_authorized`] decision. Without it the gRPC
/// listener was fully unauthenticated even with `auth.enabled = true`: any
/// client on the network could read, write, and delete documents (the listener
/// binds `server.bind_address`, which since issue #228 defaults to loopback —
/// but is `0.0.0.0` in every deployment that exposes the node at all).
///
/// The credential is read from the `authorization` request metadata
/// (`ApiKey <key>` or `Bearer <key>`), mirroring the HTTP `Authorization`
/// header. When auth is disabled (or no admin key is configured — e.g.
/// `--insecure` / first run) the interceptor is a no-op, matching the HTTP
/// surface exactly.
///
/// Note: the tonic listener speaks plaintext h2c (TLS terminates at a reverse
/// proxy or the in-process TLS on the REST/ES listeners), so in an untrusted
/// network the port should still sit behind TLS termination — but it is no
/// longer an unauthenticated open door.
#[derive(Clone)]
struct GrpcAuth {
    state: AppState,
}

impl tonic::service::Interceptor for GrpcAuth {
    fn call(&mut self, mut request: Request<()>) -> Result<Request<()>, Status> {
        let auth_header = request
            .metadata()
            .get("authorization")
            .and_then(|v| v.to_str().ok());

        // Resolve the identity, not just a yes/no, and carry it into the RPC:
        // the index an RPC touches lives in its message body, so the per-index
        // decision cannot be made here — only the principal can be.
        let principal = xerj_api::auth::authenticate(&self.state, auth_header);
        if matches!(principal, Principal::Denied) {
            return Err(Status::unauthenticated(
                "missing or invalid API key in authorization metadata",
            ));
        }
        request.extensions_mut().insert(principal);
        // Authentication deliberately runs first: an unauthenticated caller
        // must not learn whether this node rejected persisted metadata. Once
        // identity is established, reject every RPC centrally before tonic
        // dispatches to a handler. Individual handlers used to translate the
        // same blocked Engine error into not-found, invalid-argument, or a
        // partial bulk response depending on the method.
        if !self.state.engine.cluster_state_boot_status().is_writable() {
            return Err(Status::unavailable(
                xerj_common::XerjError::cluster_state_unavailable().to_string(),
            ));
        }
        Ok(request)
    }
}

/// Serve the `XerjSearch` gRPC service on `addr` until `shutdown` resolves.
///
/// Returns `Err` if the port cannot be bound or the transport fails; callers
/// log-and-continue so a gRPC bind failure never takes the whole server down.
///
/// Every RPC is guarded by [`GrpcAuth`], which enforces the same API-key auth
/// as the REST / ES-compat listeners.
///
/// The transport is cleartext h2c — see the module docs. Whether that is
/// acceptable for `addr` is decided before this is called (`main.rs` step 5b);
/// this function does not re-check, so do not call it on a new code path
/// without carrying that decision with you.
/// Bind `addr` and serve. Kept for tests, which want the convenience of `:0`
/// and do not care about startup ordering; the server binds up front via
/// [`serve_grpc_on`] instead (issue #465).
#[cfg(test)]
pub async fn serve_grpc<F>(addr: SocketAddr, state: AppState, shutdown: F) -> anyhow::Result<()>
where
    F: std::future::Future<Output = ()> + Send + 'static,
{
    let listener =
        std::net::TcpListener::bind(addr).with_context(|| format!("gRPC: bind {addr}"))?;
    listener
        .set_nonblocking(true)
        .with_context(|| format!("gRPC: set the listener on {addr} non-blocking"))?;
    let hub = xerj_api::hub_api::HubShared::new(state.clone());
    serve_grpc_on(listener, state, hub, shutdown).await
}

/// Serve gRPC on a listener the caller already bound.
///
/// The server binds every listener before it prints its banner, so that a
/// taken port fails startup instead of being logged under a banner that
/// already claimed the port (issue #465). That requires handing the bound
/// listener in rather than letting tonic bind it here.
///
/// `hub` is the [`xerj_api::hub_api::HubShared`] the ES-compat router's
/// `/_hub/v1/*` routes were built over
/// ([`xerj_api::router::build_es_compat_router_with_hub`]): the `XerjHub`
/// service draws on the same per-client rate-limit budget, query slots and
/// catalogue cache, so a second listener is not a second allowance.
pub async fn serve_grpc_on<F>(
    listener: std::net::TcpListener,
    state: AppState,
    hub: std::sync::Arc<xerj_api::hub_api::HubShared>,
    shutdown: F,
) -> anyhow::Result<()>
where
    F: std::future::Future<Output = ()> + Send + 'static,
{
    let addr = listener
        .local_addr()
        .context("gRPC: read the bound address")?;
    // `TcpIncoming::from_listener`, NOT a bare `TcpListenerStream`.
    //
    // `serve_with_incoming_shutdown` "discards any provided Server TCP
    // configuration" — including `Server::builder()`'s `tcp_nodelay: true`,
    // which tonic applies only inside `TcpIncoming::poll_next`. Handing it a
    // raw stream left Nagle on for every accepted gRPC connection and put
    // Linux's 40 ms delayed-ACK back on unary RPCs: measured p50 0.31 ms ->
    // 40.9 ms on loopback, against an unaffected ES-compat control on the same
    // node. `main.rs` sets the same option on the REST path for the same
    // reason ("a fixed per-request latency tax on trivial reads").
    //
    // `from_listener` keeps the bind-up-front property of #465 — the listener
    // is already bound, this only adopts it — while restoring the per-accept
    // socket options.
    let listener =
        tokio::net::TcpListener::from_std(listener).context("gRPC: adopt the bound listener")?;
    let incoming = tonic::transport::server::TcpIncoming::from_listener(listener, true, None)
        .map_err(|e| anyhow::anyhow!("gRPC: drive the bound listener: {e}"))?;
    let interceptor = GrpcAuth {
        state: state.clone(),
    };
    let hub = pb::xerj_hub_server::XerjHubServer::with_interceptor(
        crate::grpc_hub::HubService::new(hub),
        interceptor.clone(),
    );
    let svc = XerjSearchServer::with_interceptor(GrpcService::new(state), interceptor);
    info!("gRPC XerjSearch + XerjHub listening on {addr}");
    tonic::transport::Server::builder()
        .add_service(svc)
        .add_service(hub)
        .serve_with_incoming_shutdown(incoming, shutdown)
        .await
        .context("gRPC transport error")?;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::pb::xerj_search_client::XerjSearchClient;
    use super::*;
    use tempfile::TempDir;
    use xerj_common::config::Config;
    use xerj_common::metrics::Metrics;
    use xerj_engine::Engine;

    fn app_state(dir: &TempDir) -> AppState {
        let mut config = Config::default();
        config.server.data_dir = dir.path().to_str().unwrap().to_string();
        let metrics = Metrics::new().expect("metrics init");
        let engine = Engine::new(config.clone()).expect("engine init");
        AppState::new(config, engine, metrics)
    }

    /// Like `app_state`, but with API-key auth enabled and a fixed admin key.
    fn app_state_with_auth(dir: &TempDir, admin_key: &str) -> AppState {
        let mut config = Config::default();
        config.server.data_dir = dir.path().to_str().unwrap().to_string();
        config.auth.enabled = true;
        config.auth.admin_api_key = admin_key.to_string();
        let metrics = Metrics::new().expect("metrics init");
        let engine = Engine::new(config.clone()).expect("engine init");
        AppState::new(config, engine, metrics)
    }

    /// Grab an ephemeral port by binding then dropping a std listener.
    fn free_port() -> u16 {
        xerj_common::loopback_test::expect_loopback(std::net::TcpListener::bind("127.0.0.1:0"))
            .local_addr()
            .unwrap()
            .port()
    }

    async fn connect(addr: &str) -> XerjSearchClient<tonic::transport::Channel> {
        // The server task needs a moment to start listening; retry briefly.
        for _ in 0..50 {
            if let Ok(c) = XerjSearchClient::connect(addr.to_string()).await {
                return c;
            }
            tokio::time::sleep(std::time::Duration::from_millis(20)).await;
        }
        panic!("gRPC client could not connect to {addr}");
    }

    fn with_authorization<T>(
        message: T,
        authorization: &tonic::metadata::MetadataValue<tonic::metadata::Ascii>,
    ) -> tonic::Request<T> {
        let mut request = tonic::Request::new(message);
        request
            .metadata_mut()
            .insert("authorization", authorization.clone());
        request
    }

    #[tokio::test]
    async fn grpc_health_index_get_search_roundtrip() {
        let dir = TempDir::new().unwrap();
        let state = app_state(&dir);

        let port = free_port();
        let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
        let (tx, rx) = tokio::sync::oneshot::channel::<()>();

        let server = tokio::spawn(serve_grpc(addr, state, async move {
            let _ = rx.await;
        }));

        let url = format!("http://127.0.0.1:{port}");
        let mut client = connect(&url).await;

        // ── Health ───────────────────────────────────────────────────────
        let health = client
            .health(pb::HealthRequest {})
            .await
            .expect("health rpc")
            .into_inner();
        assert!(!health.status.is_empty(), "health status must be set");

        // ── Index → GetDocument round-trip ───────────────────────────────
        let indexed = client
            .index(pb::IndexRequest {
                index: "grpc_test".into(),
                id: "doc1".into(),
                source_json: r#"{"title":"hello grpc","n":7}"#.into(),
            })
            .await
            .expect("index rpc")
            .into_inner();
        assert_eq!(indexed.id, "doc1");
        assert_eq!(indexed.result, "created");

        let got = client
            .get_document(pb::GetRequest {
                index: "grpc_test".into(),
                id: "doc1".into(),
            })
            .await
            .expect("get rpc")
            .into_inner();
        assert!(got.found, "indexed document must be found");
        let source: serde_json::Value = serde_json::from_str(&got.source_json).unwrap();
        assert_eq!(source["title"], "hello grpc");

        // A missing document must report not-found, not error.
        let missing = client
            .get_document(pb::GetRequest {
                index: "grpc_test".into(),
                id: "nope".into(),
            })
            .await
            .expect("get rpc (missing)")
            .into_inner();
        assert!(!missing.found);

        // ── A `rerank` block is refused by name, never silently dropped ──
        // `query_json` may be a whole request body and `parse_request`
        // ignores unknown keys, so this used to return the engine's order.
        let refused = client
            .search(pb::SearchRequest {
                index: "grpc_test".into(),
                query_json: r#"{"query":{"match":{"title":"hello"}},"rerank":{}}"#.into(),
                size: 10,
                from: 0,
            })
            .await
            .expect_err("a rerank block must be refused on gRPC");
        assert_eq!(refused.code(), tonic::Code::InvalidArgument, "{refused:?}");
        assert!(refused.message().contains("rerank"), "{refused:?}");
        assert!(refused.message().contains("_search"), "{refused:?}");

        // `"rerank": null` is the same as no `rerank` key, here as on every
        // HTTP surface. This check used to be `get("rerank").is_some()`, which
        // is true for JSON null, so gRPC alone refused what the docs call
        // "absent on every surface".
        let null_is_absent = client
            .search(pb::SearchRequest {
                index: "grpc_test".into(),
                query_json: r#"{"query":{"match":{"title":"hello"}},"rerank":null}"#.into(),
                size: 10,
                from: 0,
            })
            .await
            .expect("`rerank: null` is an absent block, not a rerank request")
            .into_inner();
        assert_eq!(
            null_is_absent.hits.first().map(|h| h.id.as_str()),
            Some("doc1")
        );

        // ── Search must see the freshly indexed (memtable) doc ───────────
        let search = client
            .search(pb::SearchRequest {
                index: "grpc_test".into(),
                query_json: r#"{"match":{"title":"hello"}}"#.into(),
                size: 10,
                from: 0,
            })
            .await
            .expect("search rpc")
            .into_inner();
        assert!(
            search.total_hits >= 1,
            "search should match the indexed doc, got {}",
            search.total_hits
        );
        assert_eq!(search.hits.first().map(|h| h.id.as_str()), Some("doc1"));

        // ── Delete ───────────────────────────────────────────────────────
        let deleted = client
            .delete_document(pb::DeleteRequest {
                index: "grpc_test".into(),
                id: "doc1".into(),
            })
            .await
            .expect("delete rpc")
            .into_inner();
        assert_eq!(deleted.result, "deleted");

        // Shut the server down cleanly.
        let _ = tx.send(());
        let _ = server.await;
    }

    /// Regression for the RC4 security blocker: with `auth.enabled = true` the
    /// gRPC listener was fully unauthenticated — any client could read, write,
    /// and delete. Every RPC must now demand a valid API key (via the
    /// `authorization` metadata), and a correct key must still work end-to-end.
    #[tokio::test]
    async fn grpc_enforces_auth_when_enabled() {
        use tonic::Code;

        let dir = TempDir::new().unwrap();
        let admin = "grpc-admin-secret";
        let state = app_state_with_auth(&dir, admin);

        let port = free_port();
        let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
        let server = tokio::spawn(serve_grpc(addr, state, async move {
            let _ = rx.await;
        }));

        let url = format!("http://127.0.0.1:{port}");
        let mut client = connect(&url).await;

        // The admin key as an `authorization` metadata value.
        let auth = format!("ApiKey {admin}");
        let key_val: tonic::metadata::MetadataValue<tonic::metadata::Ascii> = auth.parse().unwrap();

        // ── Unauthenticated calls are rejected across read/write/delete ──────
        let e = client
            .health(pb::HealthRequest {})
            .await
            .expect_err("unauth health must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "health: {e:?}");

        let e = client
            .search(pb::SearchRequest {
                index: "grpc_auth".into(),
                query_json: String::new(),
                size: 0,
                from: 0,
            })
            .await
            .expect_err("unauth search must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "search: {e:?}");

        let e = client
            .index(pb::IndexRequest {
                index: "grpc_auth".into(),
                id: "x".into(),
                source_json: r#"{"a":1}"#.into(),
            })
            .await
            .expect_err("unauth index(write) must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "index: {e:?}");

        let e = client
            .get_document(pb::GetRequest {
                index: "grpc_auth".into(),
                id: "x".into(),
            })
            .await
            .expect_err("unauth get must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "get: {e:?}");

        let e = client
            .delete_document(pb::DeleteRequest {
                index: "grpc_auth".into(),
                id: "x".into(),
            })
            .await
            .expect_err("unauth delete must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "delete: {e:?}");

        // ── A wrong key is rejected ─────────────────────────────────────────
        let mut bad = tonic::Request::new(pb::HealthRequest {});
        bad.metadata_mut()
            .insert("authorization", "ApiKey wrong-key".parse().unwrap());
        let e = client
            .health(bad)
            .await
            .expect_err("wrong key must be rejected");
        assert_eq!(e.code(), Code::Unauthenticated, "wrong-key: {e:?}");

        // ── The correct admin key works end-to-end (write → read → delete) ──
        let mut req = tonic::Request::new(pb::IndexRequest {
            index: "grpc_auth".into(),
            id: "x".into(),
            source_json: r#"{"a":1}"#.into(),
        });
        req.metadata_mut().insert("authorization", key_val.clone());
        let indexed = client.index(req).await.expect("authed index").into_inner();
        assert_eq!(indexed.result, "created");

        let mut req = tonic::Request::new(pb::GetRequest {
            index: "grpc_auth".into(),
            id: "x".into(),
        });
        req.metadata_mut().insert("authorization", key_val.clone());
        let got = client
            .get_document(req)
            .await
            .expect("authed get")
            .into_inner();
        assert!(got.found, "authed get should find the doc");

        let mut req = tonic::Request::new(pb::HealthRequest {});
        req.metadata_mut().insert("authorization", key_val.clone());
        let h = client
            .health(req)
            .await
            .expect("authed health")
            .into_inner();
        assert!(!h.status.is_empty(), "authed health status must be set");

        // Shut the server down cleanly.
        let _ = tx.send(());
        let _ = server.await;
    }

    /// A rejected persisted-metadata format leaves the process available only
    /// for HTTP liveness/readiness diagnostics. The gRPC surface has no split
    /// liveness service, so every generated XerjSearch method is unavailable.
    /// Authentication stays outside that fence: missing credentials remain
    /// UNAUTHENTICATED and a valid credential reaches the UNAVAILABLE latch.
    #[tokio::test]
    async fn grpc_cluster_state_fence_covers_all_methods_after_authentication() {
        use tonic::Code;

        let dir = TempDir::new().unwrap();
        std::fs::write(
            dir.path().join("cluster_state.json"),
            br#"{"format_version":2}"#,
        )
        .unwrap();
        let admin = "grpc-blocked-admin";
        let state = app_state_with_auth(&dir, admin);
        assert!(!state.engine.cluster_state_boot_status().is_writable());

        let port = free_port();
        let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
        let server = tokio::spawn(serve_grpc(addr, state, async move {
            let _ = rx.await;
        }));
        let url = format!("http://127.0.0.1:{port}");
        let mut client = connect(&url).await;

        macro_rules! assert_code {
            ($future:expr, $code:expr, $method:literal) => {{
                let status = $future
                    .await
                    .expect_err(concat!($method, " must be rejected"));
                assert_eq!(status.code(), $code, "{}: {status:?}", $method);
            }};
        }

        // Authentication is evaluated before the storage-format latch for
        // every unary and streaming method.
        assert_code!(
            client.health(pb::HealthRequest {}),
            Code::Unauthenticated,
            "health"
        );
        assert_code!(
            client.search(pb::SearchRequest {
                index: "blocked".into(),
                query_json: String::new(),
                size: 0,
                from: 0,
            }),
            Code::Unauthenticated,
            "search"
        );
        assert_code!(
            client.index(pb::IndexRequest {
                index: "blocked".into(),
                id: "1".into(),
                source_json: "{}".into(),
            }),
            Code::Unauthenticated,
            "index"
        );
        assert_code!(
            client.bulk_index(tokio_stream::empty::<pb::IndexRequest>()),
            Code::Unauthenticated,
            "bulk_index"
        );
        assert_code!(
            client.get_document(pb::GetRequest {
                index: "blocked".into(),
                id: "1".into(),
            }),
            Code::Unauthenticated,
            "get_document"
        );
        assert_code!(
            client.delete_document(pb::DeleteRequest {
                index: "blocked".into(),
                id: "1".into(),
            }),
            Code::Unauthenticated,
            "delete_document"
        );

        let authorization: tonic::metadata::MetadataValue<tonic::metadata::Ascii> =
            format!("ApiKey {admin}").parse().unwrap();
        assert_code!(
            client.health(with_authorization(pb::HealthRequest {}, &authorization)),
            Code::Unavailable,
            "authenticated health"
        );
        assert_code!(
            client.search(with_authorization(
                pb::SearchRequest {
                    index: "blocked".into(),
                    query_json: String::new(),
                    size: 0,
                    from: 0,
                },
                &authorization,
            )),
            Code::Unavailable,
            "authenticated search"
        );
        assert_code!(
            client.index(with_authorization(
                pb::IndexRequest {
                    index: "blocked".into(),
                    id: "1".into(),
                    source_json: "{}".into(),
                },
                &authorization,
            )),
            Code::Unavailable,
            "authenticated index"
        );
        let mut bulk = tonic::Request::new(tokio_stream::empty::<pb::IndexRequest>());
        bulk.metadata_mut()
            .insert("authorization", authorization.clone());
        assert_code!(
            client.bulk_index(bulk),
            Code::Unavailable,
            "authenticated bulk_index"
        );
        assert_code!(
            client.get_document(with_authorization(
                pb::GetRequest {
                    index: "blocked".into(),
                    id: "1".into(),
                },
                &authorization,
            )),
            Code::Unavailable,
            "authenticated get_document"
        );
        assert_code!(
            client.delete_document(with_authorization(
                pb::DeleteRequest {
                    index: "blocked".into(),
                    id: "1".into(),
                },
                &authorization,
            )),
            Code::Unavailable,
            "authenticated delete_document"
        );

        let _ = tx.send(());
        let _ = server.await;
    }

    /// Standard-alphabet base64, so the test can build the `id:secret`
    /// credential a minted key presents. `xerj_api`'s encoder is crate-private.
    fn b64(input: &str) -> String {
        const A: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
        let bytes = input.as_bytes();
        let mut out = String::new();
        for chunk in bytes.chunks(3) {
            let b = [
                chunk[0],
                *chunk.get(1).unwrap_or(&0),
                *chunk.get(2).unwrap_or(&0),
            ];
            let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32;
            out.push(A[(n >> 18) as usize & 63] as char);
            out.push(A[(n >> 12) as usize & 63] as char);
            out.push(if chunk.len() > 1 {
                A[(n >> 6) as usize & 63] as char
            } else {
                '='
            });
            out.push(if chunk.len() > 2 {
                A[n as usize & 63] as char
            } else {
                '='
            });
        }
        out
    }

    /// Mint a key straight into the engine's store and return the
    /// `authorization` metadata value a client would present.
    fn mint_key(state: &AppState, id: &str, roles: Vec<xerj_engine::rbac::Role>) -> String {
        state
            .engine
            .persist_api_key(
                id.to_string(),
                xerj_engine::engine::ApiKeyRecord::new(id, "s3cret", 0, None, roles),
            )
            .unwrap();
        format!("ApiKey {}", b64(&format!("{id}:s3cret")))
    }

    /// Issue #79 on the gRPC listener. Per-brain authorization has to hold on
    /// :8081 exactly as it does on :9200 and :8080 — these RPCs take an index
    /// name straight out of the message body, so before this the boolean auth
    /// check let any authenticated client read, forge and delete any brain's
    /// edges here even while the HTTP surfaces refused.
    #[tokio::test]
    async fn grpc_enforces_per_index_authorization() {
        use std::collections::HashSet;
        use tonic::Code;
        use xerj_engine::rbac::{Privilege, Role};

        let dir = TempDir::new().unwrap();
        let admin = "grpc-admin-secret";
        let state = app_state_with_auth(&dir, admin);

        // A key scoped to alice's brain, and a legacy key with no grants.
        let alice = mint_key(
            &state,
            "alice-key",
            vec![Role::new(
                "alice",
                [Privilege::ReadIndex, Privilege::WriteIndex]
                    .into_iter()
                    .collect::<HashSet<_>>(),
                vec![".xerj-memory-alice-edges".into()],
            )],
        );
        let legacy = mint_key(&state, "legacy-key", Vec::new());

        let port = free_port();
        let addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
        let (tx, rx) = tokio::sync::oneshot::channel::<()>();
        let server = tokio::spawn(serve_grpc(addr, state, async move {
            let _ = rx.await;
        }));
        let url = format!("http://127.0.0.1:{port}");
        let mut client = connect(&url).await;

        let with = |cred: &str| {
            let v: tonic::metadata::MetadataValue<tonic::metadata::Ascii> = cred.parse().unwrap();
            v
        };

        // Seed bob's brain as the admin, so there is something to keep apart.
        let mut req = tonic::Request::new(pb::IndexRequest {
            index: ".xerj-memory-bob-edges".into(),
            id: "e1".into(),
            source_json: r#"{"src":"doc:1","dst":"secret:2","type":"mentions"}"#.into(),
        });
        req.metadata_mut()
            .insert("authorization", with(&format!("ApiKey {admin}")));
        assert_eq!(
            client
                .index(req)
                .await
                .expect("admin seed")
                .into_inner()
                .result,
            "created"
        );

        // ── alice cannot touch bob, by any RPC ──────────────────────────────
        let mut req = tonic::Request::new(pb::SearchRequest {
            index: ".xerj-memory-bob-edges".into(),
            query_json: String::new(),
            size: 10,
            from: 0,
        });
        req.metadata_mut().insert("authorization", with(&alice));
        let e = client
            .search(req)
            .await
            .expect_err("search bob must be denied");
        assert_eq!(e.code(), Code::PermissionDenied, "search: {e:?}");

        let mut req = tonic::Request::new(pb::GetRequest {
            index: ".xerj-memory-bob-edges".into(),
            id: "e1".into(),
        });
        req.metadata_mut().insert("authorization", with(&alice));
        let e = client
            .get_document(req)
            .await
            .expect_err("get bob must be denied");
        assert_eq!(e.code(), Code::PermissionDenied, "get: {e:?}");

        let mut req = tonic::Request::new(pb::IndexRequest {
            index: ".xerj-memory-bob-edges".into(),
            id: "forged".into(),
            source_json: r#"{"src":"doc:1","dst":"attacker:payload"}"#.into(),
        });
        req.metadata_mut().insert("authorization", with(&alice));
        let e = client
            .index(req)
            .await
            .expect_err("forge into bob must be denied");
        assert_eq!(e.code(), Code::PermissionDenied, "index: {e:?}");

        let mut req = tonic::Request::new(pb::DeleteRequest {
            index: ".xerj-memory-bob-edges".into(),
            id: "e1".into(),
        });
        req.metadata_mut().insert("authorization", with(&alice));
        let e = client
            .delete_document(req)
            .await
            .expect_err("delete from bob must be denied");
        assert_eq!(e.code(), Code::PermissionDenied, "delete: {e:?}");

        // A wildcard is refused rather than expanded.
        let mut req = tonic::Request::new(pb::SearchRequest {
            index: "*".into(),
            query_json: String::new(),
            size: 10,
            from: 0,
        });
        req.metadata_mut().insert("authorization", with(&alice));
        let e = client
            .search(req)
            .await
            .expect_err("wildcard must be denied");
        assert_eq!(e.code(), Code::PermissionDenied, "wildcard: {e:?}");

        // ── alice keeps full use of her own brain ───────────────────────────
        let mut req = tonic::Request::new(pb::IndexRequest {
            index: ".xerj-memory-alice-edges".into(),
            id: "e1".into(),
            source_json: r#"{"src":"doc:1","dst":"doc:2","type":"mentions"}"#.into(),
        });
        req.metadata_mut().insert("authorization", with(&alice));
        assert_eq!(
            client
                .index(req)
                .await
                .expect("own write")
                .into_inner()
                .result,
            "created"
        );
        let mut req = tonic::Request::new(pb::GetRequest {
            index: ".xerj-memory-alice-edges".into(),
            id: "e1".into(),
        });
        req.metadata_mut().insert("authorization", with(&alice));
        assert!(
            client
                .get_document(req)
                .await
                .expect("own read")
                .into_inner()
                .found
        );

        // ── the legacy key holds nothing in the reserved namespace, but keeps
        // ── ordinary indices.
        let mut req = tonic::Request::new(pb::GetRequest {
            index: ".xerj-memory-alice-edges".into(),
            id: "e1".into(),
        });
        req.metadata_mut().insert("authorization", with(&legacy));
        let e = client
            .get_document(req)
            .await
            .expect_err("legacy key must not read a brain");
        assert_eq!(e.code(), Code::PermissionDenied, "legacy reserved: {e:?}");

        let mut req = tonic::Request::new(pb::IndexRequest {
            index: "logs-2026".into(),
            id: "d1".into(),
            source_json: r#"{"msg":"hello"}"#.into(),
        });
        req.metadata_mut().insert("authorization", with(&legacy));
        assert_eq!(
            client
                .index(req)
                .await
                .expect("legacy key keeps ordinary indices")
                .into_inner()
                .result,
            "created"
        );

        let _ = tx.send(());
        let _ = server.await;
    }

    /// A share-link guest key on the gRPC listener.
    ///
    /// The guest route allow-list (`authz::guest_route_allowed`) lives in the
    /// HTTP middleware; this listener has no route list, only the per-index
    /// decision every RPC makes. Reading the code says that is a strict subset
    /// of what a guest gets over HTTP — read on the shared index, nothing on
    /// any other, no write anywhere — and until the review of PR #947 nothing
    /// exercised it. This drives the real interceptor and the real RPC
    /// handlers, in process: no socket, so no port.
    ///
    /// `bulk_index` is not called here — its argument is a tonic `Streaming`,
    /// which only a transport can build. It authorizes each item with the same
    /// `authorize(…, WriteIndex)` call that `index` makes.
    #[tokio::test]
    // `as_caller` returns the interceptor's `tonic::Status`; same rationale as
    // the allow on `authorize`.
    #[allow(clippy::result_large_err)]
    async fn a_share_guest_key_reads_its_index_over_grpc_and_nothing_else() {
        use axum::extract::State;
        use axum::Json;
        use tonic::service::Interceptor;
        use tonic::Code;

        let dir = TempDir::new().unwrap();
        let admin = "grpc-share-guest-admin";
        let state = app_state_with_auth(&dir, admin);
        let svc = GrpcService::new(state.clone());

        // Authenticate the way the listener does, then hand the principal to
        // the typed request the way tonic hands over its extensions.
        let as_caller = |authorization: &str| -> Result<Principal, Status> {
            let mut bare = tonic::Request::new(());
            bare.metadata_mut()
                .insert("authorization", authorization.parse().unwrap());
            let passed = GrpcAuth {
                state: state.clone(),
            }
            .call(bare)?;
            Ok(passed
                .extensions()
                .get::<Principal>()
                .cloned()
                .expect("the interceptor plants the principal"))
        };
        fn with<T>(message: T, principal: &Principal) -> tonic::Request<T> {
            let mut request = tonic::Request::new(message);
            request.extensions_mut().insert(principal.clone());
            request
        }
        async fn body_of(resp: axum::response::Response) -> serde_json::Value {
            let bytes = axum::body::to_bytes(resp.into_body(), 1 << 20)
                .await
                .expect("body");
            serde_json::from_slice(&bytes).expect("json")
        }

        let owner = as_caller(&format!("ApiKey {admin}")).expect("admin key");
        for (index, id, source) in [
            (
                "casefile",
                "1",
                r#"{"body":"the deposit was not returned"}"#,
            ),
            (
                "private-diary",
                "1",
                r#"{"body":"nobody else should read this"}"#,
            ),
        ] {
            let done = svc
                .index(with(
                    pb::IndexRequest {
                        index: index.into(),
                        id: id.into(),
                        source_json: source.into(),
                    },
                    &owner,
                ))
                .await
                .expect("owner indexes")
                .into_inner();
            assert_eq!(done.result, "created");
        }

        // A real share, claimed the way the guest page claims it.
        let created = body_of(
            xerj_api::share::create_share(
                State(state.clone()),
                Principal::Superuser,
                Some(Json(serde_json::json!({"index": "casefile"}))),
            )
            .await,
        )
        .await;
        let claimed = body_of(
            xerj_api::share::claim_share(
                State(state.clone()),
                xerj_api::share::ClaimSource("grpc-test".into()),
                Some(Json(serde_json::json!({
                    "id": created["share_id"], "passcode": created["passcode"],
                }))),
            )
            .await,
        )
        .await;
        let guest_key = claimed["api_key"].as_str().expect("a guest key");
        let guest = as_caller(&format!("ApiKey {guest_key}")).expect("guest key authenticates");
        assert!(guest.is_share_guest(), "{guest:?}");

        // ── what the guest holds ────────────────────────────────────────────
        let found = svc
            .search(with(
                pb::SearchRequest {
                    index: "casefile".into(),
                    query_json: String::new(),
                    size: 10,
                    from: 0,
                },
                &guest,
            ))
            .await
            .expect("a guest searches the shared index")
            .into_inner();
        assert_eq!(found.total_hits, 1, "{found:?}");
        let got = svc
            .get_document(with(
                pb::GetRequest {
                    index: "casefile".into(),
                    id: "1".into(),
                },
                &guest,
            ))
            .await
            .expect("a guest reads a shared document")
            .into_inner();
        assert!(got.found);

        // ── every other index, by name and by pattern ───────────────────────
        for index in [
            "private-diary",
            "*",
            "_all",
            "case*",
            ".xerj_audit",
            ".xerj-memory-case-edges",
            "autoindex-catalog",
        ] {
            let e = svc
                .search(with(
                    pb::SearchRequest {
                        index: index.into(),
                        query_json: String::new(),
                        size: 10,
                        from: 0,
                    },
                    &guest,
                ))
                .await
                .expect_err("a guest must not search another index");
            assert_eq!(e.code(), Code::PermissionDenied, "search [{index}]: {e:?}");
            assert!(!e.message().contains("nobody else"), "{e:?}");
            let e = svc
                .get_document(with(
                    pb::GetRequest {
                        index: index.into(),
                        id: "1".into(),
                    },
                    &guest,
                ))
                .await
                .expect_err("a guest must not read another index");
            assert_eq!(e.code(), Code::PermissionDenied, "get [{index}]: {e:?}");
        }

        // ── no write, not even to the shared index ──────────────────────────
        for index in ["casefile", "private-diary", "brand-new"] {
            let e = svc
                .index(with(
                    pb::IndexRequest {
                        index: index.into(),
                        id: "9".into(),
                        source_json: r#"{"body":"tampered"}"#.into(),
                    },
                    &guest,
                ))
                .await
                .expect_err("a guest must not write");
            assert_eq!(e.code(), Code::PermissionDenied, "index [{index}]: {e:?}");
            let e = svc
                .delete_document(with(
                    pb::DeleteRequest {
                        index: index.into(),
                        id: "1".into(),
                    },
                    &guest,
                ))
                .await
                .expect_err("a guest must not delete");
            assert_eq!(e.code(), Code::PermissionDenied, "delete [{index}]: {e:?}");
        }
        let still = svc
            .get_document(with(
                pb::GetRequest {
                    index: "casefile".into(),
                    id: "1".into(),
                },
                &owner,
            ))
            .await
            .expect("owner reads")
            .into_inner();
        assert!(still.found && still.source_json.contains("deposit"));

        // ── a revoke ends it here too ───────────────────────────────────────
        let handle = created["handle"].as_str().expect("handle").to_string();
        let revoked = body_of(
            xerj_api::share::revoke_share(
                State(state.clone()),
                Principal::Superuser,
                axum::extract::Path(handle),
            )
            .await,
        )
        .await;
        assert_eq!(revoked["keys_invalidated"], 1, "{revoked}");
        let e = as_caller(&format!("ApiKey {guest_key}")).expect_err("a revoked guest key");
        assert_eq!(e.code(), Code::Unauthenticated, "{e:?}");
    }
}