Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

113 KB
Raw Blame
//! Native xerj REST API handlers (port 8080).

use std::time::Instant;

use axum::{
    body::Bytes,
    extract::{Path, State},
    http::StatusCode,
    response::{IntoResponse, Response},
    Json,
};
use chrono::{Datelike as _, Utc};
use serde::Deserialize;
use serde_json::Value;
use uuid::Uuid;
use xerj_common::types::{FieldConfig, IndexName, Schema};
use xerj_query::parse_request;

use xerj_engine::{EnrichTable, FieldEncodingInfo};

use crate::{
    error::{native_error, ApiError},
    responses::{
        NativeHealthResponse, NativeIndexInfo, NativeIngestResponse, NativeResponse,
        NativeSchemaResponse,
    },
    state::{AppState, IndexSettings},
};

// ─────────────────────────────────────────────────────────────────────────────
// Request bodies
// ─────────────────────────────────────────────────────────────────────────────

#[derive(Debug, Deserialize)]
pub struct CreateIndexRequest {
    pub name: String,
    #[serde(default)]
    pub settings: IndexSettings,
    #[serde(default)]
    pub fields: Vec<FieldConfig>,
}

#[derive(Debug, Deserialize)]
#[serde(untagged)]
pub enum IngestRequest {
    Single(Value),
    Batch(Vec<Value>),
}

#[derive(Debug, Deserialize)]
pub struct NativeSearchRequest {
    #[serde(default)]
    pub q: Option<String>,
    #[serde(default = "default_size")]
    pub size: usize,
    #[serde(default)]
    pub from: usize,
    #[serde(default)]
    pub query: Option<Value>,
    #[serde(default)]
    pub sort: Option<Value>,
    #[serde(default)]
    pub fields: Option<Vec<String>>,
    /// Declared only so it can be REFUSED. This struct ignores unknown keys, so
    /// without the field a `rerank` block vanished and the caller got the
    /// engine's order back under a 200, believing it had been reranked.
    #[serde(default)]
    pub rerank: Option<Value>,
    /// Same rule for the local judge's block (issue #1060): declared so it is
    /// refused by name, never silently dropped.
    #[serde(default)]
    pub judge: Option<Value>,
}

fn default_size() -> usize {
    10
}

fn native_fields_supported(fields: Option<&[String]>) -> bool {
    fields.is_none_or(|fields| {
        fields
            .iter()
            .all(|field| field == xerj_query::executor::PASSAGE_RESPONSE_FIELD)
    })
}

fn native_hit_value(hit: &xerj_query::executor::Hit) -> Value {
    let mut value = serde_json::json!({
        "_id": hit.id,
        "_score": hit.score,
        "_source": hit.source,
    });
    if let Some(passage) = &hit.passage {
        value["_passage"] = serde_json::to_value(passage).unwrap_or(Value::Null);
    }
    value
}

#[cfg(test)]
mod passage_wire_tests {
    use super::*;
    use xerj_query::executor::{Hit, PassageMatch};

    fn hit(passage: Option<PassageMatch>) -> Hit {
        Hit {
            id: "report-page".to_string(),
            score: 0.91,
            source: serde_json::json!({"company": "ACME"}),
            seq_no: None,
            version: None,
            sort: Vec::new(),
            explain: None,
            highlight: None,
            matched_queries: Vec::new(),
            passage,
        }
    }

    #[test]
    fn native_fields_accept_only_passage_pseudo_field() {
        assert!(native_fields_supported(None));
        assert!(native_fields_supported(Some(&[])));
        assert!(native_fields_supported(Some(&["_passage".to_string()])));
        assert!(!native_fields_supported(Some(&["company".to_string()])));
    }

    #[test]
    fn native_passage_wire_is_opt_in_and_structured() {
        let ordinary = native_hit_value(&hit(None));
        assert!(ordinary.get("_passage").is_none());

        let passage = PassageMatch {
            field: "content".to_string(),
            ordinal: 2,
            start_offset: 320,
            end_offset: 704,
            text: "quarterly evidence".to_string(),
            page: Some(17),
        };
        let enriched = native_hit_value(&hit(Some(passage)));
        assert_eq!(enriched["_passage"]["ordinal"], 2);
        assert_eq!(enriched["_passage"]["page"], 17);
        assert_eq!(enriched["_source"], serde_json::json!({"company": "ACME"}));
    }
}

#[derive(Debug, Deserialize)]
pub struct EvolveSchemaRequest {
    pub fields: Vec<FieldConfig>,
}

#[derive(Debug, Deserialize)]
pub struct BulkIngestRequest {
    pub docs: Vec<Value>,
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices — create index
// ─────────────────────────────────────────────────────────────────────────────

pub async fn create_index(
    State(state): State<AppState>,
    Json(req): Json<CreateIndexRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let index_name = match IndexName::new(&req.name) {
        Ok(n) => n,
        Err(e) => {
            return native_error(e, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response();
        }
    };

    let name_str = index_name.as_str().to_string();

    // Build schema from provided fields.
    let mut schema = Schema::empty();
    for field in req.fields {
        if let Err(e) = schema.add_field(field) {
            return native_error(e, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response();
        }
    }

    let create_result = if state.engine.has_cluster() {
        state
            .engine
            .replicate_index_create(xerj_engine::ReplicatedIndexSpec {
                name: name_str.clone(),
                schema,
                settings: Value::Null,
                mappings: Value::Null,
                index_settings: Value::Null,
                aliases: Value::Null,
            })
            .await
    } else {
        state.engine.create_index(&name_str, schema)
    };
    if let Err(e) = create_result {
        let xerj_err: xerj_common::XerjError = e.into();
        return native_error(
            xerj_err,
            Some(&request_id),
            started.elapsed().as_millis() as u64,
        )
        .into_response();
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({ "index": name_str, "acknowledged": true }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/indices/{name}
// ─────────────────────────────────────────────────────────────────────────────

pub async fn get_index(
    State(state): State<AppState>,
    Path(name): Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let stats = idx.stats().await;
    let schema = idx.schema().await;

    let info = NativeIndexInfo {
        name: name.clone(),
        doc_count: stats.doc_count,
        schema_version: stats.schema_version,
        field_count: schema.field_count(),
        settings: IndexSettings::default(),
        created_at: Utc::now(),
        updated_at: Utc::now(),
    };

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(info, took_ms, &request_id);
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: DELETE /v1/indices/{name}
// ─────────────────────────────────────────────────────────────────────────────

pub async fn delete_index(
    State(state): State<AppState>,
    Path(name): Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    if let Err(e) = state.engine.delete_index(&name).await {
        let xerj_err: xerj_common::XerjError = e.into();
        return native_error(
            xerj_err,
            Some(&request_id),
            started.elapsed().as_millis() as u64,
        )
        .into_response();
    }
    // RC4-W4 item 5: drop the deleted index's per-index metric label series so
    // it doesn't linger for the process lifetime.
    state.metrics.prune_index_labels(&name);

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({ "index": name, "acknowledged": true }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/docs — ingest doc(s)
// ─────────────────────────────────────────────────────────────────────────────

/// Index one native document.
///
/// With a cluster handle and a shard plan, the document is stored on the
/// primary the plan names and copied to the in-sync replicas.
/// The caller still gets the native envelope: `seq_no` is the engine's
/// 1-based value, and a successful update is HTTP 201 with `result:
/// "updated"`, the same as a local write. No plan, and
/// `[cluster] enabled = false`, index on this node and do not replicate.
async fn index_one_doc(
    state: &AppState,
    idx: &xerj_engine::Index,
    name: &str,
    doc: Value,
    request_id: &str,
    started: Instant,
) -> Result<NativeIngestResponse, Response> {
    let id = extract_or_gen_id(&doc);
    match crate::cluster_shard::dispatch_index(
        state, name, &id, &doc, "index", None, None, None, None,
    )
    .await
    {
        crate::cluster_shard::Dispatch::Done(response) => {
            let (status, body) = read_dispatch_body(response).await;
            return native_ingest_from_es(name, status, &body).map_err(|(status, body)| {
                native_from_es_error(
                    status,
                    &body,
                    request_id,
                    started.elapsed().as_millis() as u64,
                )
            });
        }
        crate::cluster_shard::Dispatch::Local => {}
    }
    match idx.index_document(Some(id), doc).await {
        Ok(resp) => {
            state.metrics.record_doc_indexed(name);
            Ok(NativeIngestResponse {
                index: name.to_string(),
                id: resp.id,
                result: resp.result,
                seq_no: resp.seq_no,
            })
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            Err(native_error(
                xerj_err,
                Some(request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response())
        }
    }
}

/// `Ok(())` when the primary applied the delete (including a document that
/// was already absent). Native delete has always answered 200 `deleted` for
/// that case. `Err` is an ES-shaped body to render as a native error.
async fn route_native_delete(
    state: &AppState,
    index: &str,
    id: &str,
) -> Option<Result<(), (StatusCode, Value)>> {
    match crate::cluster_shard::dispatch_delete(state, index, id, None, None).await {
        crate::cluster_shard::Dispatch::Local => None,
        crate::cluster_shard::Dispatch::Done(response) => {
            let (status, body) = read_dispatch_body(response).await;
            Some(native_delete_from_es(status, &body))
        }
    }
}

async fn read_dispatch_body(response: Response) -> (StatusCode, Value) {
    let status = response.status();
    let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
        .await
        .unwrap_or_default();
    let body = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
    (status, body)
}

/// Translate a primary's index response into the native ingest body.
///
/// ES `_seq_no` is the engine sequence minus one. This envelope returns the
/// engine sequence, which is what a local `index_document` puts in `seq_no`.
fn native_ingest_from_es(
    index: &str,
    status: StatusCode,
    body: &Value,
) -> Result<NativeIngestResponse, (StatusCode, Value)> {
    if status.is_success() {
        if let Some(id) = body.get("_id").and_then(Value::as_str) {
            let result = body
                .get("result")
                .and_then(Value::as_str)
                .unwrap_or("created");
            let seq_no = body
                .get("_seq_no")
                .and_then(Value::as_u64)
                .unwrap_or(0)
                .saturating_add(1);
            return Ok(NativeIngestResponse {
                index: index.to_string(),
                id: id.to_string(),
                result: result.to_string(),
                seq_no,
            });
        }
    }
    Err((status, body.clone()))
}

fn native_delete_from_es(status: StatusCode, body: &Value) -> Result<(), (StatusCode, Value)> {
    if status.is_success() {
        return Ok(());
    }
    // The ES delete of a missing document is 404 `result: not_found`, not an
    // error object. Native delete reports that as `deleted`.
    if status == StatusCode::NOT_FOUND
        && body.get("error").is_none()
        && body.get("result").and_then(Value::as_str) == Some("not_found")
    {
        return Ok(());
    }
    Err((status, body.clone()))
}

fn native_from_es_error(
    status: StatusCode,
    body: &Value,
    request_id: &str,
    took_ms: u64,
) -> Response {
    let error = body.get("error");
    let error_type = error
        .and_then(|value| value.get("type"))
        .and_then(Value::as_str)
        .unwrap_or("cluster_write_failed");
    let reason = error
        .and_then(|value| value.get("reason"))
        .and_then(Value::as_str)
        .map(str::to_owned)
        .unwrap_or_else(|| format!("document write failed with HTTP {}", status.as_u16()));
    (
        status,
        Json(crate::error::NativeErrorResponse {
            error: error_type.to_string(),
            reason,
            status: status.as_u16(),
            request_id: Some(request_id.to_string()),
            took_ms,
        }),
    )
        .into_response()
}

#[cfg(test)]
mod native_cluster_envelope {
    use super::*;

    #[test]
    fn engine_seq_no_is_the_es_wire_value_plus_one() {
        let body = serde_json::json!({"_id": "1", "result": "created", "_seq_no": 0});
        let got = native_ingest_from_es("orders", StatusCode::CREATED, &body).unwrap();
        assert_eq!(got.id, "1");
        assert_eq!(got.index, "orders");
        assert_eq!(got.result, "created");
        assert_eq!(got.seq_no, 1);
    }

    #[test]
    fn an_update_keeps_the_engine_sequence() {
        let body = serde_json::json!({"_id": "1", "result": "updated", "_seq_no": 4});
        let got = native_ingest_from_es("orders", StatusCode::OK, &body).unwrap();
        assert_eq!(got.result, "updated");
        assert_eq!(got.seq_no, 5);
    }

    #[test]
    fn a_quorum_miss_is_not_a_native_success() {
        let body = serde_json::json!({
            "error": {"type": "replica_not_acknowledged", "reason": "no"},
            "status": 503
        });
        assert!(native_ingest_from_es("orders", StatusCode::SERVICE_UNAVAILABLE, &body).is_err());
    }

    #[test]
    fn a_missing_document_delete_stays_success() {
        let body = serde_json::json!({"_index": "orders", "_id": "1", "result": "not_found"});
        assert!(native_delete_from_es(StatusCode::NOT_FOUND, &body).is_ok());
    }

    #[test]
    fn an_index_not_found_delete_is_an_error() {
        let body = serde_json::json!({
            "error": {"type": "index_not_found_exception", "reason": "no"},
            "status": 404
        });
        assert!(native_delete_from_es(StatusCode::NOT_FOUND, &body).is_err());
    }
}

pub async fn ingest_docs(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(req): Json<IngestRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    // `Single` is deserialized first, and `Value` accepts an array, so a
    // JSON array arrives here rather than as `Batch`. It is still a batch:
    // one document per element, each routed by its own `_id`.
    let docs = match req {
        IngestRequest::Single(Value::Array(docs)) | IngestRequest::Batch(docs) => docs,
        IngestRequest::Single(doc) => {
            return match index_one_doc(&state, &idx, &name, doc, &request_id, started).await {
                Ok(data) => {
                    let took_ms = started.elapsed().as_millis() as u64;
                    let response = NativeResponse::new(data, took_ms, &request_id);
                    (StatusCode::CREATED, Json(response)).into_response()
                }
                Err(response) => response,
            };
        }
    };
    let count = docs.len() as u64;
    let mut ids = Vec::with_capacity(docs.len());
    let mut last_seq_no = 0u64;

    for doc in docs {
        match index_one_doc(&state, &idx, &name, doc, &request_id, started).await {
            Ok(data) => {
                last_seq_no = data.seq_no;
                ids.push(data.id);
            }
            Err(response) => return response,
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "indexed": count,
            "ids": ids,
            "last_seq_no": last_seq_no,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/indices/{name}/docs/{id}
// ─────────────────────────────────────────────────────────────────────────────

pub async fn get_doc(
    State(state): State<AppState>,
    Path((name, id)): Path<(String, String)>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    match idx.get_document(&id).await {
        Ok(Some(source)) => {
            let took_ms = started.elapsed().as_millis() as u64;
            let resp = NativeResponse::new(
                serde_json::json!({ "index": name, "id": id, "_source": source }),
                took_ms,
                &request_id,
            );
            Json(resp).into_response()
        }
        Ok(None) => {
            let e = xerj_common::XerjError::document_not_found(&id, &name);
            native_error(e, Some(&request_id), started.elapsed().as_millis() as u64).into_response()
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: DELETE /v1/indices/{name}/docs/{id}
// ─────────────────────────────────────────────────────────────────────────────

pub async fn delete_doc(
    State(state): State<AppState>,
    Path((name, id)): Path<(String, String)>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    if let Some(routed) = route_native_delete(&state, &name, &id).await {
        return match routed {
            Ok(()) => {
                let took_ms = started.elapsed().as_millis() as u64;
                let resp = NativeResponse::new(
                    serde_json::json!({ "index": name, "id": id, "result": "deleted" }),
                    took_ms,
                    &request_id,
                );
                Json(resp).into_response()
            }
            Err((status, body)) => native_from_es_error(
                status,
                &body,
                &request_id,
                started.elapsed().as_millis() as u64,
            ),
        };
    }

    match idx.delete_document(&id).await {
        Ok(_) => {
            let took_ms = started.elapsed().as_millis() as u64;
            let resp = NativeResponse::new(
                serde_json::json!({ "index": name, "id": id, "result": "deleted" }),
                took_ms,
                &request_id,
            );
            Json(resp).into_response()
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/search
// ─────────────────────────────────────────────────────────────────────────────

pub async fn search(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(req): Json<NativeSearchRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    // RC4 W4 item 2: track in-flight searches on this path too, so the
    // `active_searches` gauge reflects native-API concurrency (decremented on
    // drop, any return below). The cumulative counters + latency histograms are
    // recorded once at completion via `record_query`.
    let _search_guard = state.metrics.active_search_guard();

    if req.rerank.is_some() || req.judge.is_some() {
        crate::rerank_stage::record_refused(&state.metrics);
        let reason = if req.rerank.is_some() {
            crate::rerank_stage::unsupported_reason("the native search API")
        } else {
            crate::judge_stage::unsupported_reason("the native search API")
        };
        let error = xerj_common::XerjError::invalid_query(reason);
        return native_error(
            error,
            Some(&request_id),
            started.elapsed().as_millis() as u64,
        )
        .into_response();
    }

    if !native_fields_supported(req.fields.as_deref()) {
        let error = xerj_common::XerjError::invalid_query(
            "native search only supports the `_passage` pseudo-field; remove `fields` and read \
             values from `_source`, request only `_passage`, or use the \
             Elasticsearch-compatible `_search` endpoint",
        );
        return native_error(
            error,
            Some(&request_id),
            started.elapsed().as_millis() as u64,
        )
        .into_response();
    }

    // Build a query body for the parser.
    let mut query_body = if let Some(q_str) = &req.q {
        serde_json::json!({
            "query": {
                "multi_match": {
                    "query": q_str,
                    "fields": ["*"]
                }
            },
            "from": req.from,
            "size": req.size
        })
    } else if let Some(query) = &req.query {
        serde_json::json!({
            "query": query,
            "from": req.from,
            "size": req.size
        })
    } else {
        serde_json::json!({
            "query": { "match_all": {} },
            "from": req.from,
            "size": req.size
        })
    };
    if let Some(fields) = &req.fields {
        query_body["fields"] = serde_json::json!(fields);
    }

    let search_req = match parse_request(&query_body) {
        Ok(r) => r,
        Err(e) => {
            let ze = xerj_common::XerjError::invalid_query(e.to_string());
            return native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response();
        }
    };

    let query_type = if req.q.is_some() {
        "multi_match"
    } else {
        crate::es_compat::top_level_query_type(&req.query)
    };
    match idx.search(&search_req).await {
        Ok(result) => {
            let took_ms = started.elapsed().as_millis() as u64;
            // RC4 W4 item 2: one call records the global counter, the per-index
            // counter, and both latency histograms (overall + by query type).
            state
                .metrics
                .record_query(&name, query_type, took_ms as f64 / 1000.0);
            // RC4-W4 item 3: the native search path must feed the slow-query
            // log too (previously ONLY the ES-compat `_search` handler did, so
            // a slow query via `/v1/indices/:name/search` was invisible to the
            // operator ring buffer and the tracing line). `maybe_record` emits
            // both, gated on the shared runtime threshold, and captures the
            // query body.
            state.engine.slow_query.maybe_record(
                &name,
                "search",
                started.elapsed(),
                result.hits.len() as u64,
                "",
                &xerj_engine::slow_query::logged_search_query(
                    query_body.get("query"),
                    query_body.get("knn"),
                ),
            );

            let hits: Vec<Value> = result.hits.iter().map(native_hit_value).collect();

            let resp = NativeResponse::new(
                serde_json::json!({
                    "total": result.total.value,
                    "hits": hits,
                }),
                took_ms,
                &request_id,
            );
            Json(resp).into_response()
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/_flush
// ─────────────────────────────────────────────────────────────────────────────

/// Flush the in-memory memtable for an index to a durable on-disk segment.
///
/// After a flush the WAL checkpoint advances, old WAL generations are pruned,
/// and the data is guaranteed to survive a restart without WAL replay overhead.
pub async fn flush_index(
    State(state): State<AppState>,
    Path(name): Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    if let Err(e) = state.engine.flush_index(&name).await {
        let xerj_err: xerj_common::XerjError = e.into();
        return native_error(
            xerj_err,
            Some(&request_id),
            started.elapsed().as_millis() as u64,
        )
        .into_response();
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({ "index": name, "flushed": true }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/health
// ─────────────────────────────────────────────────────────────────────────────

pub async fn health(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let health = state.engine.health().await;

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        NativeHealthResponse {
            status: health.status,
            index_count: health.index_count,
            total_docs: health.total_docs,
            version: env!("CARGO_PKG_VERSION").to_string(),
        },
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

/// Machine-readable, privacy-safe embedding identity for resumable ingestion.
pub async fn embedding_execution_identity(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();
    let engine = state.engine.clone();
    let result = tokio::task::spawn_blocking(move || engine.embedding_execution_identity()).await;
    match result {
        Ok(Ok(identity)) => Json(NativeResponse::new(
            identity,
            started.elapsed().as_millis() as u64,
            &request_id,
        ))
        .into_response(),
        Ok(Err(error)) => {
            tracing::warn!(error = %error, "embedding execution identity is unavailable");
            native_error(
                xerj_common::XerjError::embedding(
                    "embedding execution identity is unavailable; verify the configured embedding \
                     backend and local assets in the server logs. Asset-read failures are cached \
                     for this server lifetime; restart the server after fixing the assets",
                ),
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
        Err(error) => {
            tracing::error!(error = %error, "embedding identity worker failed");
            native_error(
                xerj_common::XerjError::embedding(
                    "embedding execution identity worker failed; inspect the server logs and \
                     restart the server before retrying",
                ),
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// k8s probes — v0.8 8-P4
//
// `/health/live` and `/health/ready` are the conventional probe endpoints
// kubelet hits.  We expose them as cheap, dependency-free handlers:
//
// - **liveness**: returns 200 if the process can serve HTTP at all.  No
//   engine call, no allocations.  kubelet uses this to decide whether to
//   restart the pod.  A liveness 5xx triggers a pod restart, so this MUST
//   be a no-op — checking the engine here would create a feedback loop
//   where a slow engine flapping flush causes constant pod restarts.
//
// - **readiness**: returns 200 when the node can serve.  kubelet uses this
//   to decide whether to send traffic.  Until ready, the Service removes the
//   pod from rotation but doesn't restart it — appropriate for "still
//   replaying WAL" or "still loading 50 K segments" startup states that are
//   transient but visible.
//
//   Readiness is deliberately NOT "cluster status != red" (issue #206). One
//   index that will not open turns the engine red forever, and a pod with 200
//   healthy indices and one broken one was therefore pulled out of service
//   permanently and never came back — a single-index problem escalated into a
//   total outage, with the 199 working indices unreachable. A node is ready
//   when it has at least one index it can serve, or nothing to serve at all;
//   it is unready only when every index it holds is broken. The red status
//   itself is untouched and still reported by `/v1/health`,
//   `/v1/cluster/health`, `_cluster/health` and `_cat/health`, and the
//   degraded state is named in this body so a probe log says which it is.
// ─────────────────────────────────────────────────────────────────────────────

pub async fn liveness() -> impl IntoResponse {
    (axum::http::StatusCode::OK, "live").into_response()
}

pub async fn readiness(State(state): State<AppState>) -> impl IntoResponse {
    if !state.engine.cluster_state_boot_status().is_writable() {
        return (
            axum::http::StatusCode::SERVICE_UNAVAILABLE,
            "not ready: persisted cluster metadata was not accepted at boot; inspect server logs",
        )
            .into_response();
    }
    let h = state.engine.health().await;
    // Read the node's own state, not a principal's filtered view: this path is
    // exempt from authentication so a kubelet can probe a hardened node, and
    // the probe has to answer for the whole node. For the same reason the body
    // carries COUNTS ONLY — naming the failed indices here would hand every
    // index name on the node to anyone who can reach the port. The names live
    // behind auth on `GET /_cluster/indices/failed`.
    let failed = state.engine.failed_indices.len();
    if failed == 0 {
        return (axum::http::StatusCode::OK, "ready").into_response();
    }
    if h.index_count == 0 {
        // Nothing on this node can be served: every index it holds failed to
        // open. Removing it from rotation is correct here.
        return (
            axum::http::StatusCode::SERVICE_UNAVAILABLE,
            format!(
                "not ready: no index could be opened ({failed} failed); \
                 see GET /_cluster/indices/failed"
            ),
        )
            .into_response();
    }
    (
        axum::http::StatusCode::OK,
        format!(
            "ready (degraded): {} of {} indices serving, {failed} failed to open; \
             see GET /_cluster/indices/failed",
            h.index_count,
            h.index_count + failed
        ),
    )
        .into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET  /_cluster/indices/failed              — list failed indices
// Handler: POST /_cluster/indices/failed/{name}/_retry — retry one open
//
// Issue #206: an index whose directory refused to open had no operator
// surface at all. These two, plus `DELETE /{index}` (which now accepts a
// failed index), are the three moves the issue asks for: inspect with the
// real reason, retry after fixing the cause, delete.
// ─────────────────────────────────────────────────────────────────────────────

pub async fn failed_indices(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();
    let failed = state.engine.list_failed_indices();
    let items: Vec<serde_json::Value> = failed
        .iter()
        .map(|f| {
            serde_json::json!({
                "index": f.name,
                "reason": f.reason,
                "failed_at_millis": f.failed_at_ms,
                "retries": f.retries,
                "retry": format!("POST /_cluster/indices/failed/{}/_retry", f.name),
                "delete": format!("DELETE /{}", f.name),
            })
        })
        .collect();
    let resp = NativeResponse::new(
        serde_json::json!({ "count": items.len(), "failed_indices": items }),
        started.elapsed().as_millis() as u64,
        &request_id,
    );
    Json(resp).into_response()
}

pub async fn retry_failed_index(
    State(state): State<AppState>,
    axum::extract::Path(name): axum::extract::Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();
    match state.engine.retry_failed_index(&name) {
        Ok(()) => {
            let resp = NativeResponse::new(
                serde_json::json!({ "index": name, "reopened": true }),
                started.elapsed().as_millis() as u64,
                &request_id,
            );
            Json(resp).into_response()
        }
        // The retry did not work. Answering 200 with `reopened: false` would
        // be exactly the accepted-and-ignored shape this repo is trying to
        // stop shipping — the caller gets the live failure instead (503 for a
        // still-broken index, 404 for a name that is not a failed index).
        Err(e) => crate::error::ApiError::new(e.into()).into_response(),
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/cluster/health — native cluster health summary
//
// Mirrors the data the ES-compat `_cluster/health` exposes, in the native
// `{ data, took_ms, request_id }` envelope.
//
// Node count is the honest configured membership, identical to the ES-compat
// path (#1169): `1` when `[cluster]` is off, otherwise the distinct configured
// members (own address + peers). This is the configured floor, not live ring
// state — reachability, leader and term arrive with the ring-to-engine wiring
// tracked in #1170. A node whose cluster transport failed to bind reports the
// single node actually serving and flags `cluster_transport: "degraded"`
// (#1171), matching the ES-compat body rather than wearing the degraded state
// as a topology. Before this, the native endpoint hardcoded `1` with a comment
// claiming "single-node by design", which was false on a configured ring and
// disagreed with `_cluster/health`.
// ─────────────────────────────────────────────────────────────────────────────

pub async fn cluster_health(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let h = state.engine.health().await;
    let cluster_transport_degraded =
        state.cluster_transport == crate::state::ClusterTransportStatus::Degraded;
    let nodes = if cluster_transport_degraded {
        1
    } else {
        crate::es_compat::configured_node_count(&state.config)
    };
    let took_ms = started.elapsed().as_millis() as u64;
    let mut data = serde_json::json!({
        "cluster_name": "xerj",
        "status": h.status,
        "number_of_nodes": nodes,
        "number_of_data_nodes": nodes,
        "index_count": h.index_count,
        "total_docs": h.total_docs,
        "version": h.version,
    });
    if cluster_transport_degraded {
        data["cluster_transport"] = serde_json::json!("degraded");
    }
    let resp = NativeResponse::new(data, took_ms, &request_id);
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/admin/flush — flush every index's memtable to disk
//
// Cluster-wide counterpart of `POST /v1/indices/:name/_flush`. Walks every
// index, flushes its memtable to a durable segment, and reports per-index
// success so an operator can force a checkpoint before a backup or upgrade.
// ─────────────────────────────────────────────────────────────────────────────

pub async fn admin_flush(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let indices = state.engine.list_indices().await;
    let mut flushed: Vec<String> = Vec::new();
    let mut failed: Vec<String> = Vec::new();
    for info in &indices {
        match state.engine.flush_index(&info.name).await {
            Ok(()) => flushed.push(info.name.clone()),
            Err(_) => failed.push(info.name.clone()),
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "flushed": flushed.len(),
            "indices": flushed,
            "failed": failed,
        }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/admin/backup — snapshot the whole cluster to disk
//
// Optional JSON body: { "repo_path"?: string, "name"?: string,
// "indices"?: [string] }. Defaults: repo_path = "<data_dir>/_backups",
// name = "backup-<uuid>", indices = all. Flushes each index then copies its
// WAL + segments + schema and writes a manifest (engine::create_snapshot).
// ─────────────────────────────────────────────────────────────────────────────

#[derive(Debug, Default, Deserialize)]
pub struct BackupRequest {
    #[serde(default)]
    pub repo_path: Option<String>,
    #[serde(default)]
    pub name: Option<String>,
    #[serde(default)]
    pub indices: Option<Vec<String>>,
}

pub async fn admin_backup(State(state): State<AppState>, body: Bytes) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    // Body is optional — empty and `{}` both mean "defaults". A body that is
    // PRESENT but unparseable is not: issue #204, this was
    // `.unwrap_or_default()`, so a malformed request silently discarded the
    // caller's `repo_path` / `name` / `indices` and ran a *different* backup
    // (default location, every index) under a `201 Created`. An operator
    // reading "created" had no way to know the archive they asked for is not
    // the archive that exists.
    let req: BackupRequest = if body.is_empty() {
        BackupRequest::default()
    } else {
        match serde_json::from_slice(&body) {
            Ok(r) => r,
            Err(e) => {
                return native_error(
                    xerj_common::XerjError::invalid_document_json(format!(
                        "backup request body is not valid BackupRequest JSON: {e}"
                    )),
                    Some(&request_id),
                    started.elapsed().as_millis() as u64,
                )
                .into_response();
            }
        }
    };

    let data_dir = state.engine.config().server.data_dir.clone();
    let repo_path = req
        .repo_path
        .unwrap_or_else(|| format!("{data_dir}/_backups"));
    let name = req
        .name
        .unwrap_or_else(|| format!("backup-{}", Uuid::new_v4()));

    match state
        .engine
        .create_snapshot(&repo_path, &name, req.indices)
        .await
    {
        Ok(manifest) => {
            let took_ms = started.elapsed().as_millis() as u64;
            let resp = NativeResponse::new(
                serde_json::json!({
                    "backup": name,
                    "repo_path": repo_path,
                    "manifest": manifest,
                }),
                took_ms,
                &request_id,
            );
            (StatusCode::CREATED, Json(resp)).into_response()
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Admin: slow query log — v0.8 8-P6
//
// `GET /v1/admin/slow_queries` returns the last N slow queries.
// `DELETE /v1/admin/slow_queries` clears the buffer.
// `PUT /v1/admin/slow_queries/threshold/:ms` sets the threshold at runtime
// (also exposed via SIGHUP config reload in 8-P5).
// ─────────────────────────────────────────────────────────────────────────────

pub async fn admin_slow_queries(State(state): State<AppState>) -> impl IntoResponse {
    let log = &state.engine.slow_query;
    let body = serde_json::json!({
        "threshold_ms": log.threshold_ms(),
        "total_slow":   log.total_slow(),
        "entries":      log.snapshot(),
    });
    Json(body).into_response()
}

pub async fn admin_slow_queries_clear(State(state): State<AppState>) -> impl IntoResponse {
    state.engine.slow_query.clear();
    (axum::http::StatusCode::OK, "cleared").into_response()
}

pub async fn admin_slow_queries_set_threshold(
    State(state): State<AppState>,
    axum::extract::Path(ms): axum::extract::Path<u64>,
) -> impl IntoResponse {
    state.engine.slow_query.set_threshold_ms(ms);
    Json(serde_json::json!({ "threshold_ms": ms })).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// v0.9 9-P4 — Audit log admin
// ─────────────────────────────────────────────────────────────────────────────

/// Both handlers require [`xerj_engine::rbac::Privilege::AuditRead`]
/// (issue #329).
///
/// They took `State` and nothing else, so every authenticated credential that
/// reached the router read the whole log: a key scoped to one index — refused
/// on every other index — could read every entry on the node, security events
/// included. The privilege existed and was never consulted.
pub async fn audit_search(
    State(state): State<AppState>,
    principal: crate::auth::Principal,
) -> impl IntoResponse {
    if let Err(denied) = crate::authz::authorize_audit_read(&principal) {
        return denied;
    }
    let snap = state.engine.audit.snapshot();
    let body = serde_json::json!({
        "next_seq": state.engine.audit.next_seq(),
        "entries":  snap,
    });
    Json(body).into_response()
}

pub async fn audit_verify(
    State(state): State<AppState>,
    principal: crate::auth::Principal,
) -> impl IntoResponse {
    if let Err(denied) = crate::authz::authorize_audit_read(&principal) {
        return denied;
    }
    match state.engine.audit.verify() {
        Ok(()) => Json(serde_json::json!({ "ok": true })).into_response(),
        Err((seq, expected, actual)) => {
            let body = serde_json::json!({
                "ok": false,
                "broken_at_seq": seq,
                "expected_hash": expected,
                "actual_hash": actual,
            });
            (axum::http::StatusCode::CONFLICT, Json(body)).into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// v0.9 9-P2 — Role store admin — data model only, NOT ENFORCED (RC4 item 6)
// ─────────────────────────────────────────────────────────────────────────────

/// Honest-surface banner stamped on every `/_security/role*` response. Roles
/// are stored and round-trip, but the auth path never consults them — every
/// authenticated caller is superuser. See `xerj_engine::rbac` module docs.
const RBAC_NOT_ENFORCED_WARNING: &str =
    "roles are stored but NOT enforced: every authenticated caller has full \
     superuser access regardless of any role assignment. Full RBAC enforcement \
     is deferred.";

pub async fn rbac_list_roles(State(state): State<AppState>) -> impl IntoResponse {
    Json(serde_json::json!({
        "roles": state.engine.roles.list(),
        "enforced": false,
        "warning": RBAC_NOT_ENFORCED_WARNING,
    }))
    .into_response()
}

pub async fn rbac_get_role(
    State(state): State<AppState>,
    axum::extract::Path(name): axum::extract::Path<String>,
) -> impl IntoResponse {
    match state.engine.roles.get(&name) {
        Some(r) => Json(serde_json::json!({
            "role": r,
            "enforced": false,
            "warning": RBAC_NOT_ENFORCED_WARNING,
        }))
        .into_response(),
        None => (
            axum::http::StatusCode::NOT_FOUND,
            format!("role '{name}' not found"),
        )
            .into_response(),
    }
}

pub async fn rbac_put_role(
    State(state): State<AppState>,
    axum::extract::Path(name): axum::extract::Path<String>,
    Json(body): Json<xerj_engine::rbac::Role>,
) -> impl IntoResponse {
    let mut r = body;
    // Caller may omit the name field; the URL is authoritative.
    r.name = name;
    state.engine.roles.put(r.clone());
    // Loud signal that this PUT grants/restricts nothing (item 6).
    tracing::warn!(
        role = %r.name,
        "PUT /_security/role: role stored but NOT enforced — caller remains superuser"
    );
    Json(serde_json::json!({
        "role": r,
        "created": true,
        "enforced": false,
        "warning": RBAC_NOT_ENFORCED_WARNING,
    }))
    .into_response()
}

pub async fn rbac_delete_role(
    State(state): State<AppState>,
    axum::extract::Path(name): axum::extract::Path<String>,
) -> impl IntoResponse {
    match state.engine.roles.delete(&name) {
        Some(_) => Json(serde_json::json!({
            "deleted": true,
            "enforced": false,
            "warning": RBAC_NOT_ENFORCED_WARNING,
        }))
        .into_response(),
        None => (
            axum::http::StatusCode::NOT_FOUND,
            format!("role '{name}' not found"),
        )
            .into_response(),
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/metrics
// ─────────────────────────────────────────────────────────────────────────────

pub async fn metrics(State(state): State<AppState>) -> impl IntoResponse {
    // RC4-W4 item 4: reconcile the query-cache gauges from the engine's own
    // per-index counters at scrape time (the engine owns the truth; Prometheus
    // lives at the API layer). Cheap: a sum over the open-index set.
    let (qc_hits, qc_misses) = state.engine.query_cache_totals();
    state.metrics.set_query_cache(qc_hits, qc_misses);
    // #874: doc_count / segment_count / wal_size_bytes / memory_usage refresh
    // here too, at scrape time, the same way. A 10 s background loop used to
    // feed them; its per-tick walk of every index's WAL subtree (one stat()
    // per WAL shard, on an async runtime worker) was an idle CPU cost that
    // scales with index count — ~0.7 % of one core at 450 idle indices,
    // measured in benchmarks/idle-budget/README.md. A scrape is the only
    // moment these gauges are observable, so refreshing here keeps them live
    // exactly when they can be read and costs nothing in between. The WAL
    // walk itself runs on the blocking pool (see refresh_metric_gauges).
    crate::es_compat::refresh_metric_gauges(&state).await;
    match state.metrics.gather_text() {
        Ok(text) => (
            StatusCode::OK,
            [(
                axum::http::header::CONTENT_TYPE,
                "text/plain; version=0.0.4; charset=utf-8",
            )],
            text,
        )
            .into_response(),
        Err(e) => {
            let api_err = ApiError::new(e);
            api_err.into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/schema/{name}
// ─────────────────────────────────────────────────────────────────────────────

pub async fn get_schema(
    State(state): State<AppState>,
    Path(name): Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let schema = idx.schema().await;
    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        NativeSchemaResponse {
            index: name,
            schema,
        },
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/schema/{name}/evolve
// ─────────────────────────────────────────────────────────────────────────────

pub async fn evolve_schema(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(req): Json<EvolveSchemaRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let mut added = 0usize;
    for field in req.fields {
        match idx.add_field(field).await {
            Ok(()) => added += 1,
            Err(e) => {
                let xerj_err: xerj_common::XerjError = e.into();
                return native_error(
                    xerj_err,
                    Some(&request_id),
                    started.elapsed().as_millis() as u64,
                )
                .into_response();
            }
        }
    }

    let schema = idx.schema().await;
    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "fields_added": added,
            "schema_version": schema.version,
        }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/docs/_bulk
// ─────────────────────────────────────────────────────────────────────────────

pub async fn bulk_ingest(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(req): Json<BulkIngestRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let count = req.docs.len() as u64;
    let mut indexed = 0u64;

    for doc in req.docs {
        if index_one_doc(&state, &idx, &name, doc, &request_id, started)
            .await
            .is_ok()
        {
            indexed += 1;
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "indexed": indexed,
            "errors": indexed < count,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::OK, Json(resp)).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/logs — batch log ingestion
// ─────────────────────────────────────────────────────────────────────────────

/// Request body for the log ingest endpoint.
///
/// Accepts a JSON array of log records.  Each record is any JSON object;
/// `@timestamp`, `level`, and `message` fields are conventionally expected
/// but not required.  A unique ID is auto-generated for every record.
#[derive(Debug, Deserialize)]
pub struct LogIngestRequest(pub Vec<Value>);

pub async fn ingest_logs(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(LogIngestRequest(records)): Json<LogIngestRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    // Auto-create the index if it does not exist yet.
    let idx = match state.engine.get_or_create_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let total = records.len() as u64;
    let mut indexed = 0u64;
    let mut ids: Vec<String> = Vec::with_capacity(records.len());

    for record in records {
        // Auto-generate a unique ID for every log record.
        let id = Uuid::new_v4().to_string();
        match idx.index_document(Some(id.clone()), record).await {
            Ok(_) => {
                indexed += 1;
                ids.push(id);
                state.metrics.record_doc_indexed(&name);
            }
            Err(e) => {
                tracing::warn!(
                    index = name.as_str(),
                    error = %e,
                    "failed to index log record"
                );
            }
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "indexed": indexed,
            "total": total,
            "errors": indexed < total,
            "ids": ids,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/otlp — OpenTelemetry Log ingest
// ─────────────────────────────────────────────────────────────────────────────

/// Ingest OpenTelemetry logs in OTLP JSON format.
///
/// Expected body shape (OTLP `ExportLogsServiceRequest`):
/// ```json
/// {
///   "resourceLogs": [{
///     "resource": { "attributes": [{ "key": "service.name", "value": { "stringValue": "my-svc" } }] },
///     "scopeLogs": [{
///       "logRecords": [{
///         "timeUnixNano": "1712000000000000000",
///         "severityText": "INFO",
///         "body": { "stringValue": "hello world" },
///         "attributes": [{ "key": "http.status_code", "value": { "intValue": "200" } }]
///       }]
///     }]
///   }]
/// }
/// ```
///
/// Each log record is flattened into an indexed document with the fields:
/// `timestamp`, `severity`, `body`, `trace_id`, `span_id`, and one field per
/// OTLP attribute.  Resource attributes are merged at the top level with a
/// `resource.` prefix.
pub async fn ingest_otlp(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(body): Json<Value>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_or_create_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let mut total = 0u64;
    let mut indexed = 0u64;

    // Parse the OTLP resource-log tree.
    if let Some(resource_logs) = body.get("resourceLogs").and_then(Value::as_array) {
        for resource_log in resource_logs {
            // Collect resource-level attributes (e.g. service.name).
            let mut resource_attrs = serde_json::Map::new();
            if let Some(res) = resource_log.get("resource") {
                if let Some(attrs) = res.get("attributes").and_then(Value::as_array) {
                    for attr in attrs {
                        if let (Some(key), Some(val)) =
                            (attr.get("key").and_then(Value::as_str), attr.get("value"))
                        {
                            let field = format!("resource.{key}");
                            resource_attrs.insert(field, extract_otlp_any_value(val));
                        }
                    }
                }
            }

            if let Some(scope_logs) = resource_log.get("scopeLogs").and_then(Value::as_array) {
                for scope_log in scope_logs {
                    // Scope name (instrumentation library).
                    let scope_name = scope_log
                        .get("scope")
                        .and_then(|s| s.get("name"))
                        .and_then(Value::as_str)
                        .unwrap_or("")
                        .to_string();

                    if let Some(records) = scope_log.get("logRecords").and_then(Value::as_array) {
                        for record in records {
                            total += 1;
                            let doc = otlp_record_to_doc(record, &resource_attrs, &scope_name);
                            let id = Uuid::new_v4().to_string();
                            match idx.index_document(Some(id), doc).await {
                                Ok(_) => {
                                    indexed += 1;
                                    state.metrics.record_doc_indexed(&name);
                                }
                                Err(e) => {
                                    tracing::warn!(
                                        index = name.as_str(),
                                        error = %e,
                                        "otlp: failed to index log record"
                                    );
                                }
                            }
                        }
                    }
                }
            }
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "indexed": indexed,
            "total": total,
            "errors": indexed < total,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

/// Flatten a single OTLP `LogRecord` into a flat JSON document ready for
/// indexing.
fn otlp_record_to_doc(
    record: &Value,
    resource_attrs: &serde_json::Map<String, Value>,
    scope_name: &str,
) -> Value {
    let mut doc = serde_json::Map::new();

    // Merge resource attributes.
    for (k, v) in resource_attrs {
        doc.insert(k.clone(), v.clone());
    }

    // Instrumentation scope name.
    if !scope_name.is_empty() {
        doc.insert("scope".to_string(), Value::String(scope_name.to_string()));
    }

    // Timestamp (nanoseconds → ISO-8601 string for indexing convenience).
    if let Some(ts_nano) = record
        .get("timeUnixNano")
        .and_then(Value::as_str)
        .and_then(|s| s.parse::<u64>().ok())
    {
        let secs = (ts_nano / 1_000_000_000) as i64;
        let nanos = (ts_nano % 1_000_000_000) as u32;
        if let Some(dt) = chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nanos) {
            doc.insert("@timestamp".to_string(), Value::String(dt.to_rfc3339()));
            doc.insert("timestamp_nanos".to_string(), Value::Number(ts_nano.into()));
        }
    } else if let Some(ts_nano) = record.get("timeUnixNano").and_then(Value::as_u64) {
        let secs = (ts_nano / 1_000_000_000) as i64;
        let nanos = (ts_nano % 1_000_000_000) as u32;
        if let Some(dt) = chrono::DateTime::<chrono::Utc>::from_timestamp(secs, nanos) {
            doc.insert("@timestamp".to_string(), Value::String(dt.to_rfc3339()));
            doc.insert("timestamp_nanos".to_string(), Value::Number(ts_nano.into()));
        }
    }

    // Severity.
    if let Some(sev) = record.get("severityText").and_then(Value::as_str) {
        doc.insert("severity".to_string(), Value::String(sev.to_string()));
        // Also store numeric severity number when present.
    }
    if let Some(sev_num) = record.get("severityNumber") {
        doc.insert("severity_number".to_string(), sev_num.clone());
    }

    // Body.
    if let Some(body) = record.get("body") {
        doc.insert("message".to_string(), extract_otlp_any_value(body));
    }

    // Trace / span context.
    if let Some(tid) = record.get("traceId").and_then(Value::as_str) {
        doc.insert("trace_id".to_string(), Value::String(tid.to_string()));
    }
    if let Some(sid) = record.get("spanId").and_then(Value::as_str) {
        doc.insert("span_id".to_string(), Value::String(sid.to_string()));
    }

    // Record-level attributes.
    if let Some(attrs) = record.get("attributes").and_then(Value::as_array) {
        for attr in attrs {
            if let (Some(key), Some(val)) =
                (attr.get("key").and_then(Value::as_str), attr.get("value"))
            {
                doc.insert(key.to_string(), extract_otlp_any_value(val));
            }
        }
    }

    Value::Object(doc)
}

/// Extract a scalar Rust `Value` from an OTLP `AnyValue` object.
///
/// OTLP encodes values as `{"stringValue":"..."}`, `{"intValue":"123"}`,
/// `{"doubleValue":1.5}`, `{"boolValue":true}`, etc.
fn extract_otlp_any_value(any_val: &Value) -> Value {
    if let Some(s) = any_val.get("stringValue").and_then(Value::as_str) {
        return Value::String(s.to_string());
    }
    if let Some(i) = any_val.get("intValue") {
        // OTLP encodes int64 as a JSON string to avoid precision loss.
        if let Some(s) = i.as_str() {
            if let Ok(n) = s.parse::<i64>() {
                return Value::Number(n.into());
            }
        }
        if let Some(n) = i.as_i64() {
            return Value::Number(n.into());
        }
    }
    if let Some(d) = any_val.get("doubleValue").and_then(Value::as_f64) {
        if let Some(n) = serde_json::Number::from_f64(d) {
            return Value::Number(n);
        }
    }
    if let Some(b) = any_val.get("boolValue").and_then(Value::as_bool) {
        return Value::Bool(b);
    }
    if let Some(arr) = any_val
        .get("arrayValue")
        .and_then(|a| a.get("values"))
        .and_then(Value::as_array)
    {
        let items: Vec<Value> = arr.iter().map(extract_otlp_any_value).collect();
        return Value::Array(items);
    }
    // Fallback: return as-is.
    any_val.clone()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/syslog — Syslog line ingest
// ─────────────────────────────────────────────────────────────────────────────

/// Ingest syslog lines.
///
/// Accepts a plain-text body where each line is a syslog entry.  Supports:
///
/// * **RFC 5424** — `<priority>version timestamp hostname app-name procid msgid msg`
/// * **RFC 3164 (BSD syslog)** — `<priority>Mon DD HH:MM:SS hostname tag: message`
/// * **Plain text** — lines without a PRI are stored as bare messages.
///
/// Parsed fields: `priority`, `facility`, `severity`, `timestamp`, `hostname`,
/// `app`, `pid`, `message`.
pub async fn ingest_syslog(
    State(state): State<AppState>,
    Path(name): Path<String>,
    body: Bytes,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let body_bytes = body;

    let idx = match state.engine.get_or_create_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let text = String::from_utf8_lossy(&body_bytes);
    let mut total = 0u64;
    let mut indexed = 0u64;

    for line in text.lines() {
        let line = line.trim();
        if line.is_empty() {
            continue;
        }
        total += 1;
        let doc = parse_syslog_line(line);
        let id = Uuid::new_v4().to_string();
        match idx.index_document(Some(id), doc).await {
            Ok(_) => {
                indexed += 1;
                state.metrics.record_doc_indexed(&name);
            }
            Err(e) => {
                tracing::warn!(
                    index = name.as_str(),
                    error = %e,
                    "syslog: failed to index line"
                );
            }
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "indexed": indexed,
            "total": total,
            "errors": indexed < total,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

/// Parse a single syslog line into a flat JSON document.
///
/// Handles RFC 5424, RFC 3164, and bare message fallback.
fn parse_syslog_line(line: &str) -> Value {
    let mut doc = serde_json::Map::new();
    doc.insert("raw".to_string(), Value::String(line.to_string()));
    doc.insert(
        "@timestamp".to_string(),
        Value::String(chrono::Utc::now().to_rfc3339()),
    );

    // Try to strip the PRI field `<NNN>`.
    let (pri_opt, rest) = if line.starts_with('<') {
        if let Some(close) = line.find('>') {
            let pri_str = &line[1..close];
            let rest = &line[close + 1..];
            if let Ok(pri) = pri_str.parse::<u32>() {
                (Some(pri), rest)
            } else {
                (None, line)
            }
        } else {
            (None, line)
        }
    } else {
        (None, line)
    };

    if let Some(pri) = pri_opt {
        let facility = pri >> 3;
        let severity = pri & 0x07;
        doc.insert("priority".to_string(), Value::Number(pri.into()));
        doc.insert("facility".to_string(), Value::Number(facility.into()));
        doc.insert("severity_code".to_string(), Value::Number(severity.into()));
        doc.insert(
            "severity".to_string(),
            Value::String(syslog_severity_name(severity).to_string()),
        );
        doc.insert(
            "facility_name".to_string(),
            Value::String(syslog_facility_name(facility).to_string()),
        );
    }

    // Detect RFC 5424 vs RFC 3164 by checking whether the next field is a
    // version digit (RFC 5424) or a month abbreviation / timestamp (RFC 3164).
    let trimmed = rest.trim();

    // RFC 5424: `<VERSION> <TIMESTAMP> <HOSTNAME> <APP-NAME> <PROCID> <MSGID> <MSG>`
    // Version is a single digit (usually "1").
    if let Some(rest5424) = try_parse_rfc5424(trimmed, &mut doc) {
        doc.insert("message".to_string(), Value::String(rest5424.to_string()));
    } else if let Some(rest3164) = try_parse_rfc3164(trimmed, &mut doc) {
        doc.insert("message".to_string(), Value::String(rest3164.to_string()));
    } else {
        // Plain text fallback.
        doc.insert("message".to_string(), Value::String(trimmed.to_string()));
    }

    Value::Object(doc)
}

/// Attempt RFC 5424 parse: `1 TIMESTAMP HOSTNAME APP PROCID MSGID MSG`.
///
/// Returns the message portion on success, `None` if not RFC 5424.
fn try_parse_rfc5424<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -> Option<&'a str> {
    let mut parts = s.splitn(7, ' ');
    let version = parts.next()?;
    // RFC 5424 version is a small integer (typically "1").
    if version.parse::<u32>().is_err() {
        return None;
    }
    let timestamp = parts.next().unwrap_or("-");
    let hostname = parts.next().unwrap_or("-");
    let app = parts.next().unwrap_or("-");
    let procid = parts.next().unwrap_or("-");
    let _msgid = parts.next().unwrap_or("-");
    let msg = parts.next().unwrap_or("");

    if timestamp != "-" {
        doc.insert(
            "@timestamp".to_string(),
            Value::String(timestamp.to_string()),
        );
    }
    if hostname != "-" {
        doc.insert("hostname".to_string(), Value::String(hostname.to_string()));
    }
    if app != "-" {
        doc.insert("app".to_string(), Value::String(app.to_string()));
    }
    if procid != "-" {
        doc.insert("pid".to_string(), Value::String(procid.to_string()));
    }
    doc.insert(
        "syslog_format".to_string(),
        Value::String("rfc5424".to_string()),
    );

    // Strip optional structured data block `[...]` from the start of msg.
    let msg = msg.trim_start();
    let msg = if msg.starts_with('[') {
        if let Some(end) = msg.find(']') {
            msg[end + 1..].trim()
        } else {
            msg
        }
    } else if let Some(rest) = msg.strip_prefix('-') {
        rest.trim()
    } else {
        msg
    };

    Some(msg)
}

/// Attempt RFC 3164 parse: `Mon DD HH:MM:SS HOSTNAME TAG: MESSAGE`.
///
/// Returns the message portion on success, `None` if not RFC 3164.
fn try_parse_rfc3164<'a>(s: &'a str, doc: &mut serde_json::Map<String, Value>) -> Option<&'a str> {
    // Month abbreviations used in BSD syslog.
    const MONTHS: &[&str] = &[
        "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec",
    ];

    // The first token should be a month abbreviation.
    let first_space = s.find(' ')?;
    let month_str = &s[..first_space];
    if !MONTHS.contains(&month_str) {
        return None;
    }

    let rest = s[first_space..].trim_start();

    // Day (1 or 2 digits, possibly space-padded).
    let day_end = rest.find(' ')?;
    let day_str = rest[..day_end].trim();
    let rest = rest[day_end..].trim_start();

    // HH:MM:SS
    let time_end = rest.find(' ').unwrap_or(rest.len());
    let time_str = &rest[..time_end];
    let rest = if time_end < rest.len() {
        rest[time_end..].trim_start()
    } else {
        ""
    };

    // Build a timestamp string.
    let year = chrono::Utc::now().year();
    let ts = format!("{month_str} {day_str} {time_str} {year}");
    doc.insert("@timestamp".to_string(), Value::String(ts));
    doc.insert(
        "syslog_format".to_string(),
        Value::String("rfc3164".to_string()),
    );

    // HOSTNAME
    let hostname_end = rest.find(' ').unwrap_or(rest.len());
    let hostname = &rest[..hostname_end];
    let rest = if hostname_end < rest.len() {
        rest[hostname_end..].trim_start()
    } else {
        ""
    };
    if !hostname.is_empty() {
        doc.insert("hostname".to_string(), Value::String(hostname.to_string()));
    }

    // TAG (optional, ends at ':' or '[').
    let tag_end = rest.find([':', '[']).unwrap_or(rest.len());
    let tag = &rest[..tag_end];
    let rest = &rest[tag_end..];

    // Extract optional PID from tag[pid].
    if let Some(pid_start) = tag.find('[') {
        let app = &tag[..pid_start];
        if !app.is_empty() {
            doc.insert("app".to_string(), Value::String(app.to_string()));
        }
        if let Some(pid_end) = tag.find(']') {
            let pid = &tag[pid_start + 1..pid_end];
            doc.insert("pid".to_string(), Value::String(pid.to_string()));
        }
    } else if !tag.is_empty() {
        doc.insert("app".to_string(), Value::String(tag.to_string()));
    }

    // Message follows the ':' separator.
    let msg = rest.trim_start_matches(':').trim();
    Some(msg)
}

/// Map a syslog severity code (0–7) to a human-readable name.
fn syslog_severity_name(sev: u32) -> &'static str {
    match sev {
        0 => "emergency",
        1 => "alert",
        2 => "critical",
        3 => "error",
        4 => "warning",
        5 => "notice",
        6 => "informational",
        7 => "debug",
        _ => "unknown",
    }
}

/// Map a syslog facility code (0–23) to a human-readable name.
fn syslog_facility_name(fac: u32) -> &'static str {
    match fac {
        0 => "kern",
        1 => "user",
        2 => "mail",
        3 => "daemon",
        4 => "auth",
        5 => "syslog",
        6 => "lpr",
        7 => "news",
        8 => "uucp",
        9 => "cron",
        10 => "authpriv",
        11 => "ftp",
        16 => "local0",
        17 => "local1",
        18 => "local2",
        19 => "local3",
        20 => "local4",
        21 => "local5",
        22 => "local6",
        23 => "local7",
        _ => "unknown",
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────

fn extract_or_gen_id(doc: &Value) -> String {
    doc.get("_id")
        .and_then(Value::as_str)
        .map(str::to_owned)
        .unwrap_or_else(|| Uuid::new_v4().to_string())
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/turbo-ingest
// ─────────────────────────────────────────────────────────────────────────────

/// High-throughput batch ingest using the turbo pipeline.
///
/// Accepts a JSON array of documents and processes them using parallel
/// tokenisation and a single batched WAL write.  The turbo pipeline is
/// significantly faster than calling the single-document ingest endpoint
/// in a loop for large batches.
///
/// # Opt-in methods
///
/// 1. **Dedicated endpoint** — `POST /v1/indices/{name}/turbo-ingest`
///    with a JSON array body.
/// 2. **Header on `_bulk`** — add `X-Turbo: true` to a standard
///    `POST /v1/indices/{name}/docs/_bulk` request.
///
/// # Request body
///
/// ```json
/// [
///   { "title": "Document one", "body": "..." },
///   { "_id": "custom-id", "title": "Document two" }
/// ]
/// ```
///
/// # Response
///
/// ```json
/// {
///   "data": {
///     "index": "my-index",
///     "indexed": 1000,
///     "errors": 0,
///     "ids": ["uuid-1", "uuid-2", "..."]
///   },
///   "took_ms": 42,
///   "request_id": "..."
/// }
/// ```
pub async fn turbo_ingest(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(docs): Json<Vec<Value>>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    if docs.is_empty() {
        let took_ms = started.elapsed().as_millis() as u64;
        let resp = NativeResponse::new(
            serde_json::json!({
                "index": name,
                "indexed": 0,
                "errors": 0,
                "ids": [],
            }),
            took_ms,
            &request_id,
        );
        return (StatusCode::OK, Json(resp)).into_response();
    }

    // Pull turbo settings from engine config.
    let cfg = state.engine.config();
    let parallel = cfg.indexing.turbo_parallel;
    let fast_analyzer = cfg.indexing.turbo_fast_analyzer;

    // The engine accepts one parsed authority and serializes it exactly once
    // at the storage boundary; there is no second caller-provided byte
    // representation that could disagree with live indexing.
    let pairs: Vec<(String, Value)> = docs
        .into_iter()
        .map(|doc| {
            let id = extract_or_gen_id(&doc);
            (id, doc)
        })
        .collect();

    let total = pairs.len();

    match idx.index_batch_turbo(pairs, parallel, fast_analyzer).await {
        Ok(responses) => {
            let indexed = responses.len();
            let errors = total - indexed;
            let ids: Vec<String> = responses.into_iter().map(|r| r.id).collect();

            for _ in 0..indexed {
                state.metrics.record_doc_indexed(&name);
            }

            let took_ms = started.elapsed().as_millis() as u64;
            let resp = NativeResponse::new(
                serde_json::json!({
                    "index": name,
                    "indexed": indexed,
                    "errors": errors,
                    "ids": ids,
                }),
                took_ms,
                &request_id,
            );
            (StatusCode::OK, Json(resp)).into_response()
        }
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/indices/{name}/encodings
// ─────────────────────────────────────────────────────────────────────────────

/// Return per-field smart encoding analysis for an index.
///
/// Shows which encoding was automatically selected for each field after
/// 1 000+ samples were observed, along with the estimated compression ratio.
///
/// Fields that have not yet accumulated enough samples are omitted.
pub async fn get_index_encodings(
    State(state): State<AppState>,
    Path(name): Path<String>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let stats = idx.stats().await;

    // Sort by compression_ratio descending so the most efficiently encoded
    // fields appear first.
    let mut encodings: Vec<FieldEncodingInfo> = stats.field_encodings;
    encodings.sort_by(|a, b| {
        b.compression_ratio
            .partial_cmp(&a.compression_ratio)
            .unwrap_or(std::cmp::Ordering::Equal)
    });

    let total_fields = encodings.len();
    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "analyzed_fields": total_fields,
            "encodings": encodings,
        }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/dashboard/summary
//
// First step toward a built-in UI — returns an overview of all indices with
// doc counts, measured on-disk + memtable sizes, per-field encodings, and
// health status.
// This is intentionally designed as a "one-shot" endpoint so a lightweight
// dashboard can render a complete picture with a single HTTP request.
// ─────────────────────────────────────────────────────────────────────────────

/// Per-index summary entry in the dashboard response.
#[derive(Debug, serde::Serialize)]
pub struct DashboardIndexSummary {
    pub name: String,
    pub doc_count: u64,
    /// Real measured size in bytes: sum of on-disk segment file sizes plus the
    /// in-memory memtable byte size (the same figures reported by the
    /// `_segments` API and `IndexStats`).
    pub size_bytes: u64,
    /// Number of distinct fields.
    pub field_count: usize,
    /// Top field encodings (by compression ratio), up to 5.
    pub top_encodings: Vec<FieldEncodingInfo>,
    /// Health colour: "green" (fully flushed), "yellow" (memtable data present).
    pub health: &'static str,
}

pub async fn dashboard_summary(State(state): State<AppState>) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let mut summaries: Vec<DashboardIndexSummary> = Vec::new();
    let mut total_docs = 0u64;
    let mut overall_health = "green";

    let index_list = state.engine.list_indices().await;
    for info in &index_list {
        let idx = match state.engine.get_index(&info.name) {
            Ok(i) => i,
            Err(_) => continue,
        };
        let stats = idx.stats().await;

        let health = if stats.segment_count == 0 && stats.memtable_doc_count > 0 {
            overall_health = "yellow";
            "yellow"
        } else {
            "green"
        };

        // Real measured size: sum of on-disk segment file sizes (the same
        // per-segment `size_bytes` reported by the `_segments` API) plus the
        // in-memory memtable byte size (as reported by `IndexStats`).
        let segment_bytes: u64 = idx
            .store_snapshot()
            .segments
            .iter()
            .map(|s| s.size_bytes)
            .sum();
        let size_bytes = segment_bytes + stats.memtable_size_bytes as u64;

        let mut encodings = stats.field_encodings;
        encodings.sort_by(|a, b| {
            b.compression_ratio
                .partial_cmp(&a.compression_ratio)
                .unwrap_or(std::cmp::Ordering::Equal)
        });
        encodings.truncate(5);

        total_docs += stats.doc_count;
        summaries.push(DashboardIndexSummary {
            name: stats.name,
            doc_count: stats.doc_count,
            size_bytes,
            field_count: stats.field_count,
            top_encodings: encodings,
            health,
        });
    }

    // Sort alphabetically so the UI renders consistently.
    summaries.sort_by(|a, b| a.name.cmp(&b.name));

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index_count": summaries.len(),
            "total_doc_count": total_docs,
            "overall_health": overall_health,
            "indices": summaries,
        }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/enrich — register an enrich table
// ─────────────────────────────────────────────────────────────────────────────

/// Register (or replace) an enrichment lookup table on an index.
///
/// # Request body
///
/// ```json
/// {
///   "name": "product_meta",
///   "match_field": "sku",
///   "lookup": {
///     "SKU-001": { "category": "electronics", "brand": "Acme" },
///     "SKU-002": { "category": "clothing",    "brand": "Zephyr" }
///   }
/// }
/// ```
///
/// After registration, any document indexed into this index that contains
/// `"sku": "SKU-001"` will automatically have `"category"` and `"brand"` merged
/// in before the document is written to the WAL/FTS memtable.
///
/// # Duplicate keys
/// If a table with the same `name` already exists it is replaced atomically.
pub async fn enrich_index(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(body): Json<serde_json::Value>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    // Parse the table name, match_field, and lookup map.
    let table_name = match body.get("name").and_then(|v| v.as_str()) {
        Some(n) => n.to_string(),
        None => {
            return native_error(
                xerj_common::XerjError::invalid_query("enrich request must include a 'name' field"),
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let match_field = match body.get("match_field").and_then(|v| v.as_str()) {
        Some(f) => f.to_string(),
        None => {
            return native_error(
                xerj_common::XerjError::invalid_query(
                    "enrich request must include a 'match_field' field",
                ),
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let lookup_raw = match body.get("lookup").and_then(|v| v.as_object()) {
        Some(o) => o.clone(),
        None => {
            return native_error(
                xerj_common::XerjError::invalid_query(
                    "enrich request must include a 'lookup' object",
                ),
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let lookup: std::collections::HashMap<String, serde_json::Value> =
        lookup_raw.into_iter().collect();

    let entry_count = lookup.len();
    let table = EnrichTable {
        name: table_name.clone(),
        match_field,
        lookup,
    };

    {
        let mut enrichments = idx.enrichments.write().await;
        enrichments.insert(table_name.clone(), table);
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "enrich_table": table_name,
            "entries": entry_count,
            "acknowledged": true,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: GET /v1/indices/{name}/explain-plan — return query execution plan
// ─────────────────────────────────────────────────────────────────────────────

/// Return the query execution plan for a query WITHOUT executing it.
///
/// Shows:
/// - Which engine will be used: FTS, vector, or doc-scan
/// - Estimated cost (relative)
/// - Fields accessed
///
/// # Request body
/// Same as `_search` query body.
pub async fn explain_plan(
    State(state): State<AppState>,
    Path(name): Path<String>,
    body: Bytes,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    // Ensure the index exists.
    let idx = match state.engine.get_index(&name) {
        Ok(i) => i,
        Err(e) => {
            let xerj_err: xerj_common::XerjError = e.into();
            return native_error(
                xerj_err,
                Some(&request_id),
                started.elapsed().as_millis() as u64,
            )
            .into_response();
        }
    };

    let body_val: serde_json::Value = if body.is_empty() {
        serde_json::json!({ "query": { "match_all": {} } })
    } else {
        match serde_json::from_slice(&body) {
            Ok(v) => v,
            Err(e) => {
                return native_error(
                    xerj_common::XerjError::invalid_query(format!("invalid JSON: {e}")),
                    Some(&request_id),
                    started.elapsed().as_millis() as u64,
                )
                .into_response();
            }
        }
    };

    // Parse the query node.
    let search_req = match xerj_query::parse_request(&body_val)
        .map_err(|e| xerj_common::XerjError::invalid_query(e.to_string()))
    {
        Ok(r) => r,
        Err(e) => {
            return native_error(e, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response();
        }
    };

    let stats = idx.stats().await;
    let doc_count = stats.doc_count;

    // Analyse the query to produce an explain plan.
    let plan = build_explain_plan(&search_req.query, doc_count);

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "doc_count": doc_count,
            "plan": plan,
        }),
        took_ms,
        &request_id,
    );
    Json(resp).into_response()
}

/// Build a human-readable explain plan for a query node.
fn build_explain_plan(query: &xerj_query::ast::QueryNode, doc_count: u64) -> serde_json::Value {
    use xerj_query::ast::QueryNode;

    match query {
        QueryNode::MatchAll => serde_json::json!({
            "engine": "doc-scan",
            "description": "match_all — full collection scan",
            "estimated_docs_scanned": doc_count,
            "estimated_cost": "O(N)",
            "fields_accessed": [],
        }),
        QueryNode::MatchNone => serde_json::json!({
            "engine": "none",
            "description": "match_none — returns immediately with zero results",
            "estimated_docs_scanned": 0,
            "estimated_cost": "O(1)",
            "fields_accessed": [],
        }),
        QueryNode::Match {
            field, query: q, ..
        } => serde_json::json!({
            "engine": "fts",
            "description": format!("match query on field '{field}' for '{q}'"),
            "estimated_docs_scanned": doc_count / 10,
            "estimated_cost": "O(log N + k)",
            "fields_accessed": [field],
        }),
        QueryNode::Term { field, .. } => serde_json::json!({
            "engine": "fts",
            "description": format!("term filter on field '{field}'"),
            "estimated_docs_scanned": doc_count / 20,
            "estimated_cost": "O(log N + k)",
            "fields_accessed": [field],
        }),
        QueryNode::Range { field, .. } => serde_json::json!({
            "engine": "doc-scan",
            "description": format!("range filter on field '{field}'"),
            "estimated_docs_scanned": doc_count / 4,
            "estimated_cost": "O(N)",
            "fields_accessed": [field],
        }),
        QueryNode::Knn { field, k, .. } => serde_json::json!({
            "engine": "vector",
            "description": format!("k-NN vector search on field '{field}', k={k}"),
            "estimated_docs_scanned": k,
            "estimated_cost": "O(log N)",
            "fields_accessed": [field],
        }),
        QueryNode::Bool {
            must,
            should,
            filter,
            must_not,
            ..
        } => {
            let sub_plans: Vec<_> = must
                .iter()
                .chain(should.iter())
                .chain(filter.iter())
                .chain(must_not.iter())
                .map(|c| build_explain_plan(c, doc_count))
                .collect();
            serde_json::json!({
                "engine": "bool",
                "description": format!(
                    "bool query: {} must, {} should, {} filter, {} must_not",
                    must.len(), should.len(), filter.len(), must_not.len()
                ),
                "estimated_docs_scanned": doc_count / 5,
                "estimated_cost": "O(log N + k)",
                "clauses": sub_plans,
            })
        }
        _ => serde_json::json!({
            "engine": "doc-scan",
            "description": "generic query — doc-scan with post-filter",
            "estimated_docs_scanned": doc_count,
            "estimated_cost": "O(N)",
            "fields_accessed": [],
        }),
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: PUT /v1/pipelines/{name}  — create / replace a transform pipeline
// ─────────────────────────────────────────────────────────────────────────────

/// Create (or replace) a named transform pipeline from a JSON
/// [`PipelineConfig`](xerj_wasm::pipeline::PipelineConfig) body.
///
/// Example request body:
/// ```json
/// {
///   "description": "Enrich nginx logs",
///   "stages": [
///     { "type": "grok",        "config": { "field": "message", "pattern": "NGINX_COMBINED" } },
///     { "type": "timestamp_parse", "config": { "field": "time_local" } },
///     { "type": "drop_field",  "config": { "fields": ["message"] } },
///     { "type": "add_field",   "config": { "field": "log_type", "value": "nginx" } }
///   ]
/// }
/// ```
pub async fn put_pipeline(
    State(state): State<AppState>,
    Path(name): Path<String>,
    Json(body): Json<Value>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    // `GET /_ingest/pipeline/{name}` is the only read endpoint for a pipeline
    // created here (this surface registers no native GET), and it answers in
    // Elasticsearch's `processors` vocabulary. PUTting that output back here
    // used to hit `serde`'s "missing field `stages`" and surface as a bare
    // `500 internal error` — a client-shaped mistake reported as an engine
    // fault, with no hint of where the body belongs. Name the endpoint that
    // accepts it instead (issue #204).
    if body.get("stages").is_none() && body.get("processors").is_some() {
        let e = xerj_common::XerjError::config(format!(
            "pipeline [{name}]: this body is Elasticsearch-shaped (`processors`), which \
             this endpoint does not accept — it takes xerj's `stages` form. `GET \
             /_ingest/pipeline/{name}` renders every stored pipeline in Elasticsearch \
             vocabulary, so PUT that body to `/_ingest/pipeline/{name}` instead."
        ));
        return native_error(e, Some(&request_id), started.elapsed().as_millis() as u64)
            .into_response();
    }

    // This surface rejects a definition it cannot compile, so nothing is
    // stored on a compile error. On success the pipeline is written to
    // `<data_dir>/cluster_state.json` before we acknowledge it (issue #203):
    // it used to live only in memory and vanish on restart, exactly like the
    // ES-compat one.
    match state.engine.put_pipeline(&name, body) {
        Ok(None) => {
            let took_ms = started.elapsed().as_millis() as u64;
            let resp = NativeResponse::new(
                serde_json::json!({ "pipeline": name, "acknowledged": true }),
                took_ms,
                &request_id,
            );
            (StatusCode::OK, Json(resp)).into_response()
        }
        // The definition does not compile — same 500 this surface has always
        // returned, and nothing was stored.
        Ok(Some(compile_err)) => {
            let ze = xerj_common::XerjError::internal(compile_err.to_string());
            native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response()
        }
        Err(e) => {
            let ze = xerj_common::XerjError::internal(e.to_string());
            native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response()
        }
    }
}

// ─────────────────────────────────────────────────────────────────────────────
// Handler: POST /v1/indices/{name}/ingest?pipeline={pipeline}
// ─────────────────────────────────────────────────────────────────────────────

/// Query parameters for the pipeline-ingest endpoint.
#[derive(Debug, Deserialize)]
pub struct IngestWithPipelineParams {
    pub pipeline: String,
}

/// Ingest one or more documents through a named transform pipeline before
/// indexing.
///
/// Documents with [`ProcessAction::Drop`] are silently discarded.
/// Documents with [`ProcessAction::Route(target)`] are indexed into `target`
/// instead of `{name}`.
pub async fn ingest_with_pipeline(
    State(state): State<AppState>,
    Path(name): Path<String>,
    axum::extract::Query(params): axum::extract::Query<IngestWithPipelineParams>,
    Json(req): Json<IngestRequest>,
) -> impl IntoResponse {
    let started = Instant::now();
    let request_id = Uuid::new_v4().to_string();

    let docs: Vec<Value> = match req {
        IngestRequest::Single(doc) => vec![doc],
        IngestRequest::Batch(docs) => docs,
    };

    let processed = match state
        .engine
        .process_through_pipeline(&params.pipeline, docs)
    {
        Ok(p) => p,
        Err(e) => {
            let ze = xerj_common::XerjError::internal(e.to_string());
            return native_error(ze, Some(&request_id), started.elapsed().as_millis() as u64)
                .into_response();
        }
    };

    let mut indexed = 0u64;
    let mut dropped = 0u64;
    let mut routed = 0u64;
    let mut errors = 0u64;

    for (action, doc) in processed {
        use xerj_wasm::pipeline::ProcessAction;
        let target_name = match action {
            ProcessAction::Drop => {
                dropped += 1;
                continue;
            }
            ProcessAction::Route(ref target) => {
                routed += 1;
                target.clone()
            }
            ProcessAction::Pass => name.clone(),
        };

        let idx = match state.engine.get_or_create_index(&target_name) {
            Ok(i) => i,
            Err(_) => {
                errors += 1;
                continue;
            }
        };

        let doc_id = extract_or_gen_id(&doc);
        match idx.index_document(Some(doc_id), doc).await {
            Ok(_) => {
                indexed += 1;
                state.metrics.record_doc_indexed(&target_name);
            }
            Err(_) => {
                errors += 1;
            }
        }
    }

    let took_ms = started.elapsed().as_millis() as u64;
    let resp = NativeResponse::new(
        serde_json::json!({
            "index": name,
            "pipeline": params.pipeline,
            "indexed": indexed,
            "dropped": dropped,
            "routed": routed,
            "errors": errors,
        }),
        took_ms,
        &request_id,
    );
    (StatusCode::CREATED, Json(resp)).into_response()
}

#[cfg(test)]
mod cluster_health_topology_tests {
    //! Issue #1170 (honest positioning) / #1169: the native
    //! `GET /v1/cluster/health` used to hardcode `number_of_nodes: 1` with a
    //! "single-node by design" comment, disagreeing with the ES-compat
    //! `_cluster/health` after that path learned the configured membership.
    //! These pin the two bodies to the same honest count.
    use crate::state::AppState;
    use axum::{
        body::Body,
        http::{Request, StatusCode},
    };
    use serde_json::Value;
    use tower::ServiceExt;

    fn state_with_cluster(enabled: bool, peers: &[&str], cluster_port: u16) -> AppState {
        let dir = tempfile::tempdir().expect("tempdir").keep();
        let mut config = xerj_common::config::Config::default();
        config.server.data_dir = dir.to_string_lossy().into_owned();
        config.storage.wal_sync = xerj_common::config::WalSync::Async;
        config.server.bind_address = "127.0.0.1".to_string();
        if enabled {
            config.cluster.enabled = true;
            config.cluster.port = cluster_port;
            config.cluster.auth_secret = "test-secret-0123456789abcdef".to_string();
            config.cluster.peers = peers.iter().map(|s| s.to_string()).collect();
        }
        let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
        let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
        AppState::new(config, engine, metrics)
    }

    async fn get(app: &axum::Router, path: &str) -> (StatusCode, Value) {
        let response = app
            .clone()
            .oneshot(Request::get(path).body(Body::empty()).unwrap())
            .await
            .unwrap();
        let status = response.status();
        let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
            .await
            .unwrap();
        (status, serde_json::from_slice(&bytes).unwrap())
    }

    #[tokio::test]
    async fn single_node_reports_one() {
        let state = state_with_cluster(false, &[], 9300);
        let app = crate::router::build_native_router(state);
        let (status, body) = get(&app, "/v1/cluster/health").await;
        assert_eq!(status, StatusCode::OK);
        assert_eq!(body["data"]["number_of_nodes"], 1);
        assert_eq!(body["data"]["number_of_data_nodes"], 1);
        assert!(body["data"].get("cluster_transport").is_none());
    }

    #[tokio::test]
    async fn configured_ring_reports_real_membership() {
        let state = state_with_cluster(
            true,
            &[
                "127.0.0.1:19311=127.0.0.1:19311",
                "127.0.0.1:19312=127.0.0.1:19312",
            ],
            19310,
        );
        let app = crate::router::build_native_router(state);
        let (status, body) = get(&app, "/v1/cluster/health").await;
        assert_eq!(status, StatusCode::OK);
        // Own address + two distinct peers = 3, never a hardcoded 1 (#1170).
        assert_eq!(body["data"]["number_of_nodes"], 3);
        assert_eq!(body["data"]["number_of_data_nodes"], 3);
    }

    #[tokio::test]
    async fn self_in_peer_list_is_not_double_counted() {
        let state = state_with_cluster(
            true,
            &[
                "127.0.0.1:19310=127.0.0.1:19310",
                "127.0.0.1:19311=127.0.0.1:19311",
            ],
            19310,
        );
        let app = crate::router::build_native_router(state);
        let (_, body) = get(&app, "/v1/cluster/health").await;
        assert_eq!(body["data"]["number_of_nodes"], 2);
    }
}