AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(rank_eval): compute real dcg/nDCG/mrr/err instead of returning precision
xerj-orgc036f0835128
diff --git a/docs/recipes/semantic-search-rag.md b/docs/recipes/semantic-search-rag.mdindex d3a0c3e41..7f70b34a3 100644--- a/docs/recipes/semantic-search-rag.md+++ b/docs/recipes/semantic-search-rag.md@@ -186,3 +186,39 @@ the KB, runs the five semantic retrievals with assertions, prints the RAG contexbundle, shows the keyword-vs-semantic contrast, demonstrates the lexical-embeddermiss, and exits non-zero if any of the five retrievals regress. Point `BASE` at anyXERJ (or real Elasticsearch with a `semantic_text`/inference setup) to compare.++---++## Measuring ranking quality with `_rank_eval`++Once you have relevance judgements (which docs *should* come back for a+query), the `_rank_eval` API scores your ranking with the standard IR+metrics — the same wire API as Elasticsearch. XERJ computes each metric for+real (they are **not** aliases for precision): `precision`, `recall`, `dcg`+(add `"normalize": true` for nDCG), `mean_reciprocal_rank`, and+`expected_reciprocal_rank`. An unknown metric name returns `400` rather than+a misleading number.++```bash+curl -sX POST "$BASE/kb/_rank_eval" -H 'content-type: application/json' -d '{+ "metric": { "dcg": { "k": 10, "normalize": true } },+ "requests": [+ {+ "id": "chunking_q",+ "request": { "query": { "semantic": { "field": "content",+ "query": "how do I split documents for RAG?", "k": 10 } } },+ "ratings": [+ { "_id": "kb-7", "rating": 3 },+ { "_id": "kb-12", "rating": 1 },+ { "_id": "kb-30", "rating": 0 }+ ]+ }+ ]+}'+# → { "metric_score": 0.87, "details": { "chunking_q": { "metric_score": 0.87, ... } } }+```++Swap `"dcg"` for `"precision"`, `"recall"`, `"mean_reciprocal_rank"`, or+`"expected_reciprocal_rank"` to score the same ranking a different way — each+returns its own value, so you can track whichever metric your product cares+about as you tune embeddings, `k`, and hybrid weights.diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rsindex eebb4ac58..72b4e3d95 100644--- a/engine/crates/xerj-api/src/es_compat.rs+++ b/engine/crates/xerj-api/src/es_compat.rs@@ -22338,23 +22338,205 @@ pub struct RankEvalRating {pub rating: u32,}+/// Real per-request rank_eval metric score. `ranked` is the graded relevance+/// of each retrieved doc in rank order; `all_ratings` is every rating supplied+/// for the request (used for recall's denominator, nDCG's ideal ordering, and+/// ERR's default max grade). Pure so it can be unit-tested against known values.+fn rank_eval_metric_score(+ metric: &str,+ mparams: &Value,+ ranked: &[u32],+ all_ratings: &[u32],+ k: usize,+ threshold: u32,+ normalize: bool,+) -> f64 {+ match metric {+ "precision" => {+ let rel = ranked.iter().filter(|&&r| r >= threshold).count();+ if ranked.is_empty() {+ 0.0+ } else {+ rel as f64 / ranked.len() as f64+ }+ }+ "recall" => {+ let total_rel = all_ratings.iter().filter(|&&r| r >= threshold).count();+ let rel = ranked.iter().filter(|&&r| r >= threshold).count();+ if total_rel == 0 {+ 0.0+ } else {+ rel as f64 / total_rel as f64+ }+ }+ // DCG = Σ (2^rating − 1) / log2(rank + 1), rank 1-based.+ // nDCG divides by the ideal DCG (ratings sorted descending, top-k).+ "dcg" => {+ let dcg = |gains: &[u32]| -> f64 {+ gains+ .iter()+ .enumerate()+ .map(|(i, &g)| (2f64.powi(g as i32) - 1.0) / ((i as f64) + 2.0).log2())+ .sum()+ };+ let raw = dcg(ranked);+ if normalize {+ let mut ideal: Vec<u32> = all_ratings.to_vec();+ ideal.sort_unstable_by(|a, b| b.cmp(a));+ ideal.truncate(k);+ let idcg = dcg(&ideal);+ if idcg > 0.0 {+ raw / idcg+ } else {+ 0.0+ }+ } else {+ raw+ }+ }+ // Reciprocal rank of the first doc at/above the relevance threshold.+ "mean_reciprocal_rank" => ranked+ .iter()+ .position(|&r| r >= threshold)+ .map(|p| 1.0 / (p as f64 + 1.0))+ .unwrap_or(0.0),+ // ERR = Σ_i (1/i)·(Π_{j<i}(1−R_j))·R_i, R_g = (2^g − 1)/2^max_grade.+ "expected_reciprocal_rank" => {+ let max_grade = mparams+ .get("maximum_relevance")+ .and_then(Value::as_u64)+ .map(|v| v as u32)+ .unwrap_or_else(|| all_ratings.iter().copied().max().unwrap_or(1))+ .max(1);+ let denom = 2f64.powi(max_grade as i32);+ let mut err = 0.0;+ let mut p_stay = 1.0;+ for (i, &g) in ranked.iter().enumerate() {+ let r = (2f64.powi(g as i32) - 1.0) / denom;+ err += p_stay * r / (i as f64 + 1.0);+ p_stay *= 1.0 - r;+ }+ err+ }+ _ => 0.0,+ }+}++#[cfg(test)]+mod rank_eval_metric_tests {+ use super::rank_eval_metric_score;+ use serde_json::json;++ // ranked ratings [3,0,1] retrieved in that order; all ratings [3,0,1].+ // Each metric must produce a DISTINCT, correct value (the bug was that+ // dcg/mrr/err all silently returned precision).+ const RANKED: [u32; 3] = [3, 0, 1];+ const ALL: [u32; 3] = [3, 0, 1];++ fn score(metric: &str, params: serde_json::Value, normalize: bool) -> f64 {+ rank_eval_metric_score(metric, ¶ms, &RANKED, &ALL, 3, 1, normalize)+ }++ #[test]+ fn precision_is_relevant_over_retrieved() {+ assert!((score("precision", json!({}), false) - 2.0 / 3.0).abs() < 1e-9);+ }++ #[test]+ fn recall_is_relevant_retrieved_over_total() {+ assert!((score("recall", json!({}), false) - 1.0).abs() < 1e-9);+ }++ #[test]+ fn dcg_uses_graded_gain_and_log_discount() {+ // 7/log2(2) + 0/log2(3) + 1/log2(4) = 7 + 0 + 0.5+ assert!((score("dcg", json!({}), false) - 7.5).abs() < 1e-9);+ }++ #[test]+ fn ndcg_normalizes_against_ideal() {+ // ideal [3,1,0] → 7 + 1/log2(3) = 7.630930; ndcg = 7.5 / 7.630930+ assert!((score("dcg", json!({}), true) - (7.5 / (7.0 + 1.0 / 3f64.log2()))).abs() < 1e-9);+ }++ #[test]+ fn mrr_is_reciprocal_of_first_relevant_rank() {+ // first rating>=1 is at rank 1 → 1.0+ assert!((score("mean_reciprocal_rank", json!({}), false) - 1.0).abs() < 1e-9);+ }++ #[test]+ fn err_uses_cascade_probabilities() {+ // max_grade 3, denom 8; R=[7/8,0,1/8] → 0.875 + 0 + (1/3)(0.125)(0.125)+ let expected = 0.875 + (1.0 / 3.0) * 0.125 * 0.125;+ assert!((score("expected_reciprocal_rank", json!({"maximum_relevance":3}), false)+ - expected)+ .abs()+ < 1e-9);+ }++ #[test]+ fn non_precision_metrics_do_not_collapse_to_precision() {+ // The original bug: dcg / nDCG / mrr / err all silently returned the+ // precision value. Each must now differ from precision.+ let precision = score("precision", json!({}), false);+ for (name, val) in [+ ("dcg", score("dcg", json!({}), false)),+ ("ndcg", score("dcg", json!({}), true)),+ ("mrr", score("mean_reciprocal_rank", json!({}), false)),+ ("err", score("expected_reciprocal_rank", json!({"maximum_relevance":3}), false)),+ ] {+ assert!(+ (val - precision).abs() > 1e-6,+ "{name} collapsed to precision ({precision})"+ );+ }+ }+}+pub async fn rank_eval(State(state): State<AppState>,Path(index): Path<String>,Json(body): Json<RankEvalBody>,) -> impl IntoResponse {let mut details: serde_json::Map<String, Value> = serde_json::Map::new();- let mut all_precision: Vec<f64> = Vec::new();- let mut all_recall: Vec<f64> = Vec::new();+ let mut all_scores: Vec<f64> = Vec::new();- // Default metric: precision at 10.- let k = body+ // The metric object carries exactly one key naming the metric. Default+ // to precision when omitted (back-compat). Each metric computes its own+ // real score below — an unknown metric fails loud (400) rather than+ // silently returning precision.+ let (metric_name, mparams): (String, Value) = body.metric.as_ref()- .and_then(|m| m.get("precision").or_else(|| m.get("recall")))- .and_then(|m| m.get("k"))+ .and_then(|m| m.as_object())+ .and_then(|o| o.iter().next().map(|(k, v)| (k.clone(), v.clone())))+ .unwrap_or_else(|| ("precision".to_string(), json!({})));++ const KNOWN_METRICS: [&str; 5] = [+ "precision",+ "recall",+ "dcg",+ "mean_reciprocal_rank",+ "expected_reciprocal_rank",+ ];+ if !KNOWN_METRICS.contains(&metric_name.as_str()) {+ return ApiError::new(xerj_common::XerjError::invalid_query(format!(+ "unknown rank_eval metric [{metric_name}]; supported: precision, recall, dcg, \+ mean_reciprocal_rank, expected_reciprocal_rank"+ )))+ .into_response();+ }++ let k = mparams.get("k").and_then(Value::as_u64).unwrap_or(10) as usize;+ let threshold = mparams+ .get("relevant_rating_threshold").and_then(Value::as_u64)- .unwrap_or(10) as usize;+ .unwrap_or(1) as u32;+ let normalize = mparams+ .get("normalize")+ .and_then(Value::as_bool)+ .unwrap_or(false);for req_spec in &body.requests {let query_val = req_spec@@ -22384,38 +22566,34 @@ pub async fn rank_eval(Err(_) => continue,};- // Build set of relevant doc ids (rating >= 1).- let relevant_ids: std::collections::HashSet<String> = req_spec- .ratings- .iter()- .filter(|r| r.rating >= 1)- .map(|r| r.id.clone())- .collect();-let retrieved_ids: Vec<String> = result.hits.iter().take(k).map(|h| h.id.clone()).collect();- let relevant_retrieved: usize = retrieved_ids- .iter()- .filter(|id| relevant_ids.contains(*id))- .count();-- let precision = if retrieved_ids.is_empty() {- 0.0- } else {- relevant_retrieved as f64 / retrieved_ids.len() as f64- };-- let recall = if relevant_ids.is_empty() {- 1.0- } else {- relevant_retrieved as f64 / relevant_ids.len() as f64+ // Rating of each retrieved doc, in rank order (unrated → 0).+ let rating_of = |id: &str| -> u32 {+ req_spec+ .ratings+ .iter()+ .find(|r| r.id == id)+ .map(|r| r.rating)+ .unwrap_or(0)};+ let ranked: Vec<u32> = retrieved_ids.iter().map(|id| rating_of(id)).collect();+ let all_ratings: Vec<u32> = req_spec.ratings.iter().map(|r| r.rating).collect();++ let score = rank_eval_metric_score(+ &metric_name,+ &mparams,+ &ranked,+ &all_ratings,+ k,+ threshold,+ normalize,+ );- all_precision.push(precision);- all_recall.push(recall);+ all_scores.push(score);details.insert(req_spec.id.clone(), json!({- "metric_score": precision,+ "metric_score": score,"unrated_docs": [],"hits": retrieved_ids.iter().enumerate().map(|(i, id)| {let rating = req_spec.ratings.iter().find(|r| &r.id == id).map(|r| r.rating as i64).unwrap_or(-1);@@ -22428,14 +22606,16 @@ pub async fn rank_eval(}));}- let mean_precision = if all_precision.is_empty() {+ // Top-level score is the mean across requests (ES aggregates every+ // rank_eval metric by averaging per-request scores).+ let mean_score = if all_scores.is_empty() {0.0} else {- all_precision.iter().sum::<f64>() / all_precision.len() as f64+ all_scores.iter().sum::<f64>() / all_scores.len() as f64};Json(json!({- "metric_score": mean_precision,+ "metric_score": mean_score,"details": details,"failures": {}}))