Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

11.6 KB
Raw Blame
//! Slow query log — v0.8 8-P6.
//!
//! When a search exceeds a configurable wall-clock threshold (default
//! 1 second), record a structured entry into a bounded ring buffer.
//! The most-recent N entries can be retrieved via the admin endpoint
//! `GET /v1/admin/slow_queries` so an operator can debug "why is this
//! cluster suddenly slow?" without enabling a full debug log.
//!
//! Design notes:
//! - Bounded by capacity (default 256) — we never grow unbounded under
//!   a runaway slow-query workload.
//! - Lock-free reads: the buffer is a single `parking_lot::RwLock` and
//!   readers take a shared lock to clone the snapshot.
//! - Append is O(1): grow until capacity, then overwrite oldest.
//! - Threshold is per-process global; per-index overrides can be added
//!   later if needed (most cluster-debug scenarios want a single knob).
//! - Each entry carries the query body, bounded to [`SLOW_QUERY_MAX_BODY`]
//!   chars so an operator can see *what* was slow — not just that something
//!   was. The cap keeps the ring safe to serialize over the admin endpoint
//!   even at full capacity (256 × 2 KB ≈ 512 KB worst case).
//! - This is the single source of truth for slow-query observability: the
//!   `tracing` warn/error line is emitted from inside [`maybe_record`] using
//!   the SAME runtime threshold as the ring buffer, so the log and the buffer
//!   can never disagree and both track the hot-reloadable knob.
//!
//! A future enhancement will export this as a Prometheus histogram + counter
//! (`xerj_search_slow_total`) so the operator can graph slow-query rates per
//! index in Grafana.

use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;

/// Default threshold: 1 second.
pub const DEFAULT_SLOW_QUERY_MS: u64 = 1_000;
/// Default ring buffer capacity.
pub const DEFAULT_SLOW_QUERY_CAPACITY: usize = 256;
/// Max chars of the query body retained per entry (and emitted on the tracing
/// line). Keeps the bounded ring small and safe to serialize even at full
/// capacity, and stops a pathological multi-megabyte query DSL from bloating
/// either the buffer or a log line.
pub const SLOW_QUERY_MAX_BODY: usize = 2_048;

/// Truncate a query body to [`SLOW_QUERY_MAX_BODY`] chars on a UTF-8 char
/// boundary, appending an ellipsis when clipped.
fn truncate_body(s: &str) -> String {
    if s.len() <= SLOW_QUERY_MAX_BODY {
        return s.to_string();
    }
    let mut end = SLOW_QUERY_MAX_BODY;
    while end > 0 && !s.is_char_boundary(end) {
        end -= 1;
    }
    format!("{}…", &s[..end])
}

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SlowQueryEntry {
    /// Wall-clock time of the request, milliseconds since epoch.
    pub at_ms: u64,
    /// Index name (or "_all", "_search").
    pub index: String,
    /// Query type / route (e.g. "search", "search/agg", "knn").
    pub op: String,
    /// Total wall-clock duration in milliseconds.
    pub took_ms: u64,
    /// Number of hits (or 0 for size:0 / aggs-only).
    pub hits: u64,
    /// Optional short reason (e.g. "many segments scanned", "agg fan-out").
    pub note: String,
    /// The query body (bounded to [`SLOW_QUERY_MAX_BODY`] chars). Search
    /// callers fill this via [`logged_search_query`], so an omitted clause
    /// is recorded as `{"match_all":{}}` (what the engine runs) and a
    /// top-level `knn` is recorded with the query instead of an empty
    /// `query=` field. This is what makes an entry *actionable* — the
    /// operator sees the exact DSL that ran slow.
    pub query: String,
}

/// Text stored in [`SlowQueryEntry::query`] for one search.
///
/// An omitted query is match_all. A top-level `knn` with no query clause is
/// the knn object, not an empty string — an empty `query=` on the tracing
/// line reads as "the body was lost". Null clauses are treated as absent.
/// When both are present the entry keeps both, because logging only `query`
/// hides the knn that may be why the search was slow.
pub fn logged_search_query(
    query: Option<&serde_json::Value>,
    knn: Option<&serde_json::Value>,
) -> String {
    let query = query.filter(|q| !q.is_null());
    let knn = knn.filter(|k| !k.is_null());
    match (query, knn) {
        (Some(q), None) => q.to_string(),
        (Some(q), Some(knn)) => serde_json::json!({"query": q, "knn": knn}).to_string(),
        (None, Some(knn)) => serde_json::json!({"knn": knn}).to_string(),
        (None, None) => r#"{"match_all":{}}"#.to_string(),
    }
}

/// Bounded ring buffer of slow query entries.
pub struct SlowQueryLog {
    buf: RwLock<Vec<SlowQueryEntry>>,
    capacity: usize,
    threshold_ms: AtomicU64,
    /// Total slow events ever observed (for metrics / sanity).
    total: AtomicU64,
}

impl SlowQueryLog {
    pub fn new(capacity: usize, threshold_ms: u64) -> Arc<Self> {
        Arc::new(Self {
            buf: RwLock::new(Vec::with_capacity(capacity)),
            capacity,
            threshold_ms: AtomicU64::new(threshold_ms),
            total: AtomicU64::new(0),
        })
    }

    /// Threshold getter (atomic).
    pub fn threshold_ms(&self) -> u64 {
        self.threshold_ms.load(Ordering::Relaxed)
    }

    /// Threshold setter (atomic) — can be hot-reloaded from config.
    pub fn set_threshold_ms(&self, ms: u64) {
        self.threshold_ms.store(ms, Ordering::Relaxed);
    }

    /// Total slow events observed since process start.
    pub fn total_slow(&self) -> u64 {
        self.total.load(Ordering::Relaxed)
    }

    /// If `took >= threshold`, record an entry AND emit the slow-query
    /// tracing line. No-op otherwise. Returns `true` if the entry was
    /// recorded.
    ///
    /// This is the ONE place slow queries are surfaced. Both the ring buffer
    /// and the `tracing` warn/error line are gated on the same runtime
    /// `threshold_ms` knob (hot-reloadable via `set_threshold_ms`), so they
    /// can never drift apart — previously the ring buffer respected the knob
    /// while `Index::search` logged against hardcoded 1s/5s thresholds.
    /// `error!` fires at 5× the knob, `warn!` at 1×.
    pub fn maybe_record(
        &self,
        index: &str,
        op: &str,
        took: Duration,
        hits: u64,
        note: &str,
        query: &str,
    ) -> bool {
        let took_ms = took.as_millis() as u64;
        let threshold = self.threshold_ms.load(Ordering::Relaxed);
        if took_ms < threshold {
            return false;
        }
        self.total.fetch_add(1, Ordering::Relaxed);
        let query = truncate_body(query);
        // Unified tracing — same threshold as the ring buffer above.
        if took_ms >= threshold.saturating_mul(5) {
            tracing::error!(took_ms, index, op, hits, query = %query, "slow query");
        } else {
            tracing::warn!(took_ms, index, op, hits, query = %query, "slow query");
        }
        let entry = SlowQueryEntry {
            at_ms: std::time::SystemTime::now()
                .duration_since(std::time::UNIX_EPOCH)
                .map(|d| d.as_millis() as u64)
                .unwrap_or(0),
            index: index.to_string(),
            op: op.to_string(),
            took_ms,
            hits,
            note: note.to_string(),
            query,
        };
        let mut buf = self.buf.write();
        if buf.len() < self.capacity {
            buf.push(entry);
        } else {
            // Rotate: drop oldest, push newest.  Over-large capacities
            // are rare so the O(N) shift is acceptable; if this ever
            // matters we can switch to a true ring with head index.
            buf.remove(0);
            buf.push(entry);
        }
        true
    }

    /// Snapshot the current buffer (cheap clone of N small structs).
    pub fn snapshot(&self) -> Vec<SlowQueryEntry> {
        self.buf.read().clone()
    }

    /// Clear the buffer (e.g. after operator inspection).
    pub fn clear(&self) {
        self.buf.write().clear();
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::time::Duration;

    #[test]
    fn under_threshold_no_record() {
        let log = SlowQueryLog::new(8, 1000);
        let recorded = log.maybe_record("idx", "search", Duration::from_millis(500), 10, "", "");
        assert!(!recorded);
        assert!(log.snapshot().is_empty());
        assert_eq!(log.total_slow(), 0);
    }

    #[test]
    fn over_threshold_records() {
        let log = SlowQueryLog::new(8, 1000);
        let recorded = log.maybe_record(
            "idx",
            "search",
            Duration::from_millis(1500),
            10,
            "test",
            r#"{"match_all":{}}"#,
        );
        assert!(recorded);
        let snap = log.snapshot();
        assert_eq!(snap.len(), 1);
        assert_eq!(snap[0].took_ms, 1500);
        assert_eq!(snap[0].index, "idx");
        // The query body is captured so the entry is actionable.
        assert_eq!(snap[0].query, r#"{"match_all":{}}"#);
        assert_eq!(log.total_slow(), 1);
    }

    #[test]
    fn ring_rotates_at_capacity() {
        let log = SlowQueryLog::new(2, 100);
        for i in 0..5 {
            log.maybe_record("idx", "search", Duration::from_millis(200 + i), i, "", "");
        }
        let snap = log.snapshot();
        assert_eq!(snap.len(), 2);
        // Oldest two were dropped; we should see the last two.
        assert_eq!(snap[0].took_ms, 203);
        assert_eq!(snap[1].took_ms, 204);
        assert_eq!(log.total_slow(), 5);
    }

    #[test]
    fn threshold_can_be_updated() {
        let log = SlowQueryLog::new(8, 1000);
        log.set_threshold_ms(50);
        assert_eq!(log.threshold_ms(), 50);
        let recorded = log.maybe_record("idx", "search", Duration::from_millis(100), 0, "", "");
        assert!(recorded);
    }

    #[test]
    fn long_query_body_is_truncated_on_char_boundary() {
        let log = SlowQueryLog::new(4, 100);
        // A multibyte body larger than the cap must be clipped without
        // panicking on a char boundary, and marked with an ellipsis.
        let big = "é".repeat(SLOW_QUERY_MAX_BODY); // 2 bytes each ⇒ well over the cap
        let recorded = log.maybe_record("idx", "search", Duration::from_millis(200), 1, "", &big);
        assert!(recorded);
        let snap = log.snapshot();
        assert_eq!(snap.len(), 1);
        let stored = &snap[0].query;
        assert!(
            stored.ends_with('…'),
            "clipped body should end with an ellipsis"
        );
        // Body (minus the ellipsis) never exceeds the byte cap.
        assert!(stored.len() <= SLOW_QUERY_MAX_BODY + '…'.len_utf8());
    }

    #[test]
    fn logged_search_query_is_never_empty() {
        let knn = serde_json::json!({"field": "embedding", "query_vector": [0.1, 0.2], "k": 10});
        let lexical = serde_json::json!({"match": {"body": "rotate keys"}});

        let omitted = logged_search_query(None, None);
        assert_eq!(omitted, r#"{"match_all":{}}"#);
        assert!(!omitted.is_empty());

        let nulls = logged_search_query(
            Some(&serde_json::Value::Null),
            Some(&serde_json::Value::Null),
        );
        assert_eq!(nulls, r#"{"match_all":{}}"#);

        assert_eq!(
            logged_search_query(Some(&lexical), None),
            lexical.to_string()
        );

        let knn_only = logged_search_query(None, Some(&knn));
        assert_eq!(knn_only, serde_json::json!({"knn": knn}).to_string());
        assert!(!knn_only.is_empty());

        let both = logged_search_query(Some(&lexical), Some(&knn));
        assert_eq!(
            both,
            serde_json::json!({"query": lexical, "knn": knn}).to_string()
        );
        assert!(both.contains("\"knn\""));
        assert!(both.contains("rotate keys"));
    }
}