Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

search: opt-in local `judge` stage on _search — _p_relevant per hit (#1060)

xerj-org321d747795d5
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 92ae58529..617c81835 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7596,9 +7596,11 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
+ "tempfile",
"thiserror 2.0.18",
"tokio",
"tracing",
+ "xerj-ai",
]
[[package]]
diff --git a/engine/crates/xerj-api/Cargo.toml b/engine/crates/xerj-api/Cargo.toml
index b6671d26b..edd24f1cd 100644
--- a/engine/crates/xerj-api/Cargo.toml
+++ b/engine/crates/xerj-api/Cargo.toml
@@ -14,8 +14,9 @@ neural = ["xerj-engine/neural"]
# decide ladder, answering /v1/systemone and /_decide when the history
# index has no support. Mirrors how `neural` is gated — off by default,
# enabled by whoever links this crate (the xerj-server wiring is a
-# follow-up).
-decide-local = ["dep:xerj-ai", "xerj-ai/decide-local"]
+# follow-up). Also arms the `judge` stage's model arm (issue #1060): the
+# same loader, the same model directory, the same loaded copy.
+decide-local = ["dep:xerj-ai", "xerj-ai/decide-local", "xerj-rerank/judge-local"]
onnx-experimental = ["xerj-engine/onnx-experimental"]
[dependencies]
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index 553bab2c4..dd9dd7f5e 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -3453,6 +3453,12 @@ pub struct EsSearchBody {
/// word would return lexical order to a caller who asked for something else.
#[serde(default)]
pub rerank: Option<Value>,
+ /// Opt-in local judging of the returned hits — see
+ /// [`crate::judge_stage`] (issue #1060). Declared for the same reason as
+ /// `rerank` one field above: a `judge` block serde dropped without a word
+ /// would hand back unjudged hits under a 200.
+ #[serde(default)]
+ pub judge: Option<Value>,
/// `post_filter` — accepted-and-ignored for ordinary queries (it never
/// reaches the engine; implementing it generally is #204), but captured
/// here because beside a `hybrid` query it must be REJECTED (#943): the
@@ -3497,6 +3503,7 @@ impl Default for EsSearchBody {
pit: None,
timeout: None,
rerank: None,
+ judge: None,
post_filter: None,
}
}
@@ -3555,6 +3562,8 @@ const ES_SEARCH_TOP_LEVEL_KEYS: &[&str] = &[
// reranking (`crate::rerank_stage`). Listed so the unknown-key gate lets it
// through to a stage that validates its contents strictly.
"rerank",
+ // The second (`crate::judge_stage`, issue #1060), same rule.
+ "judge",
];
/// A minimal position-tracking JSON scanner used solely to locate the first
@@ -7959,6 +7968,11 @@ fn search_response_hint(index: &str, body: &EsSearchBody, hits: &[EsHit]) -> Opt
if let Some(rerank) = &body.rerank {
hint["hints"][0]["try"]["body"]["rerank"] = rerank.clone();
}
+ // Same rule for the judge block: a suggested request that lost the
+ // caller's `judge` would come back unjudged and unpruned.
+ if let Some(judge) = &body.judge {
+ hint["hints"][0]["try"]["body"]["judge"] = judge.clone();
+ }
Some(hint)
}
@@ -9462,6 +9476,32 @@ async fn search_impl(
}
};
+ // Validate the `judge` block under the same discipline, before the search
+ // runs: a request whose judge block is unsatisfiable should not pay for
+ // the search that cannot be judged. Runs AFTER `rerank`'s prepare because
+ // that is the one stage allowed to widen the page; the judge judges the
+ // page as emitted and needs no widening of its own.
+ let judge_plan = match crate::judge_stage::JudgePlan::prepare(&mut body, is_scroll_request) {
+ Ok(plan) => plan,
+ Err(reason) => {
+ return (
+ StatusCode::BAD_REQUEST,
+ Json(json!({
+ "error": {
+ "root_cause": [{
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ }],
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ },
+ "status": 400,
+ })),
+ )
+ .into_response();
+ }
+ };
+
// `?typed_keys` — prefix agg names with their type in the response.
let typed_keys = params.typed_keys.is_some();
@@ -14249,6 +14289,7 @@ async fn search_impl(
),
ignored: ignored_list,
ignored_field_values,
+ p_relevant: None,
}
})
.collect();
@@ -14466,6 +14507,53 @@ async fn search_impl(
}
};
+ // ── judge stage ────────────────────────────────────────────────────────
+ // Opt-in local judging (issue #1060). Runs AFTER the rerank stage, so it
+ // judges the page this response actually emits: the two blocks compose
+ // (the hosted rerank widens and reorders its window, the local judge then
+ // re-judges and prunes the emitted page) and each stands alone. The
+ // engine's — or the hosted provider's — `_score` stays untouched on every
+ // hit; the judge adds `_p_relevant` and may drop hits below `min_p`,
+ // which is why `took` and the `_savings` sum are re-stamped here too.
+ let judged_info = match &judge_plan {
+ None => None,
+ Some(plan) => {
+ let applied = plan.apply(&state.decide, &mut hits, &mut max_score).await;
+ match applied {
+ Ok(info) => {
+ took_ms = started.elapsed().as_millis() as u64;
+ // `_savings` is summed over the hits a response emits; a
+ // `min_p` drop removed some.
+ if let Some((record, _)) = savings_claim {
+ let emitted: u64 = hits
+ .iter()
+ .filter_map(|h| savings_by_hit.get(&(h.index.clone(), h.id.clone())))
+ .sum();
+ savings_claim = Some((record, emitted));
+ }
+ Some(info)
+ }
+ Err(e) => {
+ let status = crate::judge_stage::status_for(&e);
+ let kind = crate::judge_stage::error_type_for(&e);
+ let reason = format!("judge failed: {e}");
+ return (
+ StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY),
+ Json(json!({
+ "error": {
+ "root_cause": [{ "type": kind, "reason": reason }],
+ "type": kind,
+ "reason": reason,
+ },
+ "status": status,
+ })),
+ )
+ .into_response();
+ }
+ }
+ }
+ };
+
// Computed before `hits` is moved into `response_body` below.
// Two independent diagnostics, merged into one `hints` array rather than
// one winning: a query can both name a nonexistent field AND return an
@@ -14625,6 +14713,22 @@ async fn search_impl(
// engine's own after a degrade. It goes ahead of `hits` for the same reason
// the hint below does: a reader that truncates long output from the bottom
// must still see it.
+ // `judged` says the same thing about the local judge's pass — which
+ // scorer judged, what was kept and what was dropped — and goes ahead of
+ // `hits` for the same reader-truncation reason as `_rerank` below. It is
+ // prepended FIRST so that `_rerank`'s own prepend lands it ahead of
+ // `judged`: the wire order is the order the stages ran.
+ if let Some(info) = judged_info {
+ if let Some(obj) = response_body.as_object_mut() {
+ let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);
+ rebuilt.insert("judged".to_string(), info);
+ for (k, v) in obj.iter() {
+ rebuilt.insert(k.clone(), v.clone());
+ }
+ *obj = rebuilt;
+ }
+ }
+
if let Some(info) = rerank_info {
if let Some(obj) = response_body.as_object_mut() {
let mut rebuilt = serde_json::Map::with_capacity(obj.len() + 1);
@@ -21184,10 +21288,15 @@ pub async fn search_with_scroll(
// A scroll streams the engine's order; `rerank` cannot apply to it, and
// this handler used to rebuild the body with `rerank: None` — dropping it.
- if body.rerank.is_some() {
+ // The `judge` stage reorders too, so the same refusal applies.
+ if body.rerank.is_some() || body.judge.is_some() {
return (
StatusCode::BAD_REQUEST,
- Json(crate::rerank_stage::unsupported_on("_search_scroll")),
+ Json(if body.rerank.is_some() {
+ crate::rerank_stage::unsupported_on("_search_scroll")
+ } else {
+ crate::judge_stage::unsupported_on("_search_scroll")
+ }),
)
.into_response();
}
@@ -21252,6 +21361,7 @@ pub async fn search_with_scroll(
pit: body.pit.clone(),
timeout: body.timeout.clone(),
rerank: None,
+ judge: None,
// #943: forward so the post_filter-beside-hybrid rejection in
// `build_search_request` covers scroll requests too.
post_filter: body.post_filter.clone(),
@@ -21379,6 +21489,7 @@ pub async fn search_with_scroll(
},
ignored: None,
ignored_field_values: None,
+ p_relevant: None,
}
})
.collect();
@@ -21522,6 +21633,7 @@ pub async fn search_with_scroll(
},
ignored: None,
ignored_field_values: None,
+ p_relevant: None,
})
.collect();
@@ -21556,6 +21668,10 @@ pub struct ScrollBody {
/// class the `?scroll=` path already refuses.
#[serde(default)]
pub rerank: Option<Value>,
+ /// Same rule for `judge` (issue #1060): a continuation streams the
+ /// engine's order and cannot re-judge it.
+ #[serde(default)]
+ pub judge: Option<Value>,
}
#[derive(Debug, Deserialize, Default)]
@@ -21575,10 +21691,14 @@ pub async fn next_scroll(
) -> impl IntoResponse {
// scroll_id may come from body OR query param
let body = body.into_or_default();
- if body.rerank.is_some() {
+ if body.rerank.is_some() || body.judge.is_some() {
return (
StatusCode::BAD_REQUEST,
- Json(crate::rerank_stage::unsupported_on("_search/scroll")),
+ Json(if body.rerank.is_some() {
+ crate::rerank_stage::unsupported_on("_search/scroll")
+ } else {
+ crate::judge_stage::unsupported_on("_search/scroll")
+ }),
)
.into_response();
}
@@ -21784,6 +21904,7 @@ async fn scroll_page_response(
},
ignored: None,
ignored_field_values: None,
+ p_relevant: None,
}
})
.collect();
@@ -24102,9 +24223,13 @@ async fn msearch_impl(
// `rerank` is a `_search` stage and this path has its own, smaller
// renderer. `parse_request` ignores keys it does not know, so without
// this the item came back 200 in the engine's order — refused per item
- // instead, and the rest of the batch still runs.
- if crate::rerank_stage::carries_rerank(&search_body_val) {
- responses.push(crate::rerank_stage::unsupported_on("_msearch"));
+ // instead, and the rest of the batch still runs. The `judge` stage is
+ // refused for the same reason.
+ if crate::judge_stage::carries_second_stage(&search_body_val) {
+ responses.push(crate::judge_stage::unsupported_second_stage_on(
+ "_msearch",
+ &search_body_val,
+ ));
continue;
}
@@ -32787,11 +32912,15 @@ pub async fn search_template(
}
// A template that renders a `rerank` block: this minimal path does not run
- // the stage, and `parse_request` would drop the key without a word.
- if crate::rerank_stage::carries_rerank(&search_body_val) {
+ // the stage, and `parse_request` would drop the key without a word. Same
+ // for a `judge` block.
+ if crate::judge_stage::carries_second_stage(&search_body_val) {
return (
StatusCode::BAD_REQUEST,
- Json(crate::rerank_stage::unsupported_on("_search/template")),
+ Json(crate::judge_stage::unsupported_second_stage_on(
+ "_search/template",
+ &search_body_val,
+ )),
)
.into_response();
}
@@ -32982,9 +33111,12 @@ async fn msearch_template_impl(
};
// Same refusal as `_msearch` and `_search/template`: the rendered body
- // carries a `rerank` block this path would silently drop.
- if crate::rerank_stage::carries_rerank(&search_body_val) {
- responses.push(crate::rerank_stage::unsupported_on("_msearch/template"));
+ // carries a `rerank` or `judge` block this path would silently drop.
+ if crate::judge_stage::carries_second_stage(&search_body_val) {
+ responses.push(crate::judge_stage::unsupported_second_stage_on(
+ "_msearch/template",
+ &search_body_val,
+ ));
continue;
}
@@ -37619,11 +37751,17 @@ pub async fn async_search_submit(
let mut body = body.into_or_default();
// An async search stores its response and hands it back later; a stored
// rerank would also be a stored third-party call nobody is waiting on.
- // Refused rather than dropped.
- if body.rerank.is_some() {
+ // Refused rather than dropped — and a stored `judge` pass is refused too:
+ // the caller asked for judged, pruned hits and would get the stored
+ // engine's order instead.
+ if body.rerank.is_some() || body.judge.is_some() {
return (
StatusCode::BAD_REQUEST,
- Json(crate::rerank_stage::unsupported_on("_async_search")),
+ Json(if body.rerank.is_some() {
+ crate::rerank_stage::unsupported_on("_async_search")
+ } else {
+ crate::judge_stage::unsupported_on("_async_search")
+ }),
)
.into_response();
}
@@ -38294,8 +38432,9 @@ pub async fn rank_eval(
// "reranking changed nothing", under a 200, from the one endpoint whose
// job is measuring ranking quality. Reported per request in `failures`
// (the rest of the batch still runs), like every other request this
- // handler cannot run, and nothing is sent to the provider.
- if crate::rerank_stage::carries_rerank(&req_spec.request) {
+ // handler cannot run, and nothing is sent to the provider. A `judge`
+ // block would vanish the same way, for the same reason.
+ if crate::judge_stage::carries_second_stage(&req_spec.request) {
crate::rerank_stage::record_refused(&state.metrics);
// The same `illegal_argument_exception` body every other surface
// that does not run the stage answers with, not the
@@ -38303,7 +38442,9 @@ pub async fn rank_eval(
// request was never run.
failures.insert(
req_spec.id.clone(),
- crate::rerank_stage::unsupported_on("_rank_eval")["error"].clone(),
+ crate::judge_stage::unsupported_second_stage_on("_rank_eval", &req_spec.request)
+ ["error"]
+ .clone(),
);
continue;
}
diff --git a/engine/crates/xerj-api/src/judge_stage.rs b/engine/crates/xerj-api/src/judge_stage.rs
new file mode 100644
index 000000000..df7989638
--- /dev/null
+++ b/engine/crates/xerj-api/src/judge_stage.rs
@@ -0,0 +1,727 @@
+//! The `judge` stage of `_search` — the zero-token second stage (issue #1060).
+//!
+//! Runs after the engine has produced and rendered its hits (and after the
+//! hosted `rerank` stage, when both blocks are present: the two compose, each
+//! stands alone). Unlike [`crate::rerank_stage`], this stage never leaves the
+//! node: it hands the emitted page to a [`xerj_rerank::judge::LocalJudge`] in
+//! process — no provider, no key, no tokens, no egress. That is the point of
+//! the issue's "zero-token rerank": a caller can ask "how likely is each of
+//! THESE hits to be relevant, as a number I can threshold" on every search
+//! without an operator paying for it.
+//!
+//! # What the stage owns
+//!
+//! Two things: the ORDER of `hits.hits` (judged hits first, probability
+//! descending, ties keeping the engine's order — a stable sort) and, with
+//! `judge.min_p`, the MEMBERSHIP (hits below the threshold are dropped and
+//! counted). Everything else in the response stays the engine's, exactly as
+//! the rerank stage leaves it: `hits.total`, `aggregations`, `suggest` all
+//! describe the full match set.
+//!
+//! The engine's `_score` is NOT replaced — the one deliberate difference from
+//! the hosted stage. `_p_relevant` is added to each judged hit, so a client
+//! sorting by `_score` keeps the engine's own number while a client
+//! thresholding `_p_relevant` gets an absolute `0..=1` scale `_score` never
+//! had. `hits.max_score` still describes `_score`, recomputed only because a
+//! `min_p` drop can remove the hit that carried the maximum.
+//!
+//! # Which judge judged
+//!
+//! The response's `judged` block names the scorer. A node armed with a local
+//! decision head (`XERJ_DECIDE_MODE=local`, feature `decide-local`) judges
+//! with that head as a cross-encoder; every other node judges with the
+//! deterministic lexical scorer, and SAYS SO — `judged.scorer` is `"lexical"`
+//! or `"model"`, and `judged.model` appears only on the model arm, because a
+//! response naming a "model" that is arithmetic would be the silent fake this
+//! repository refuses everywhere. No quality claim is made for either arm
+//! here: the BEIR gate in issue #1060 binds at release.
+//!
+//! # Strictness
+//!
+//! Same rules as the rerank stage, for the same reasons (see that module's
+//! header): `sort` / `search_after` / `collapse` / `scroll` are refused
+//! because the caller already fixed the order or the paging model cannot
+//! survive a reorder-and-prune; `_source: false` with nothing that puts text
+//! on a hit is refused rather than judged blind; `size: 0` is refused because
+//! there is nothing to judge. Surfaces that do not run the stage refuse a body
+//! carrying `judge` by name — dropping the block silently returns unjudged
+//! hits to a caller who asked for judged ones, the accepted-and-ignored
+//! defect class. `"judge": null` is the same as no `judge` key everywhere.
+
+use serde_json::{json, Value};
+use std::time::Instant;
+use xerj_rerank::judge::{JudgeConfig, LocalJudge};
+use xerj_rerank::{Candidate, RerankError};
+
+use crate::es_compat::EsSearchBody;
+use crate::responses::EsHit;
+use crate::systemone_api::DecideSettings;
+
+// Only the model arm names a model directory; the import (and the arm) exist
+// under the same feature that compiles the decide head.
+#[cfg(feature = "decide-local")]
+use xerj_rerank::judge::ModelJudge;
+
+/// A validated judge request, produced before the search runs.
+#[derive(Debug)]
+pub struct JudgePlan {
+ cfg: JudgeConfig,
+ query: String,
+}
+
+impl JudgePlan {
+ /// Validate the `judge` block against the request it rides on.
+ ///
+ /// Must run AFTER the URL parameters have been merged into `body`, and
+ /// after [`crate::rerank_stage::RerankPlan::prepare`] (the one stage
+ /// allowed to widen the page — the judge judges the page as emitted and
+ /// widens nothing). Returns `Ok(None)` when the body has no `judge` block;
+ /// `Err` carries the reason for a 400.
+ pub fn prepare(body: &mut EsSearchBody, scrolling: bool) -> Result<Option<Self>, String> {
+ let Some(raw) = body.judge.as_ref() else {
+ return Ok(None);
+ };
+ let cfg = JudgeConfig::from_json(raw).map_err(|e| e.to_string())?;
+
+ if scrolling {
+ return Err(
+ "`judge` cannot be combined with `scroll`: a scroll streams the engine's \
+ order page by page, and a stage that reorders and prunes a page would \
+ break the scroll's own position mid-stream"
+ .into(),
+ );
+ }
+ if body.sort.is_some() {
+ return Err(
+ "`judge` cannot be combined with `sort`: an explicit sort already fixes the \
+ order, and judging would reorder around it"
+ .into(),
+ );
+ }
+ if body.search_after.is_some() {
+ return Err("`judge` cannot be combined with `search_after`".into());
+ }
+ if body.collapse.is_some() {
+ return Err("`judge` cannot be combined with `collapse`".into());
+ }
+ // Mirrors the rerank stage's rule: `_source: false` on its own leaves
+ // no text on a hit, so there is certainly nothing to judge. Beside a
+ // clause that puts values on the hit (`fields`, `docvalue_fields`,
+ // `stored_fields`, `script_fields`) it is coherent and is let through
+ // to the check on the rendered hits in `apply`.
+ let non_empty = |v: &Option<Value>| match v {
+ Some(Value::Array(a)) => !a.is_empty(),
+ Some(Value::Object(o)) => !o.is_empty(),
+ Some(Value::String(s)) => !s.trim().is_empty(),
+ _ => false,
+ };
+ let asks_for_fields = non_empty(&body.fields)
+ || non_empty(&body.docvalue_fields)
+ || non_empty(&body.stored_fields)
+ || non_empty(&body.script_fields);
+ if matches!(body.source, Some(Value::Bool(false))) && !asks_for_fields {
+ return Err(
+ "`judge` needs document text: `_source: false` with no `fields` (or \
+ `docvalue_fields`, `stored_fields`, `script_fields`) clause leaves nothing \
+ to judge. Return the text through `fields`, or drop `_source: false`"
+ .into(),
+ );
+ }
+ if body.size == 0 {
+ return Err(
+ "`judge` cannot be combined with `size: 0`: the response returns no hits, \
+ so there is nothing to judge. Drop `judge` for an aggregation-only request, \
+ or ask for at least one hit"
+ .into(),
+ );
+ }
+
+ // The same inference rule as the rerank stage: read the question only
+ // from the shapes that carry exactly one, refuse to guess otherwise.
+ let query = match cfg.query.clone() {
+ Some(q) => q,
+ None => body
+ .query
+ .as_ref()
+ .and_then(crate::rerank_stage::infer_query)
+ .ok_or_else(|| {
+ "`judge.query` is required: the search query is not a shape a single \
+ question can be read from (supported for inference: match, match_phrase, \
+ multi_match, semantic, simple_query_string). A `bool`, a `hybrid`, a \
+ top-level `knn` with no text query, or no `query` at all needs the \
+ question spelled out"
+ .to_string()
+ })?,
+ };
+ // `judge.query` was checked against its ceiling by the parser; an
+ // inferred question is the same string on the same wire and gets the
+ // same ceiling.
+ if cfg.query.is_none() {
+ xerj_rerank::check_inferred_question(&query).map_err(|e| e.to_string())?;
+ }
+
+ Ok(Some(Self { cfg, query }))
+ }
+
+ /// Judge `hits` in place and return the `judged` block for the response.
+ ///
+ /// `Err` means a fault the caller must see (see
+ /// [`xerj_rerank::Policy`]): a config problem is a 400, a local model that
+ /// cannot load or score is a 503. There is deliberately no degrade path —
+ /// a caller who asked for judged hits and silently got unjudged ones (or
+ /// a different scorer than the response names) has been misled.
+ pub async fn apply(
+ &self,
+ decide: &DecideSettings,
+ hits: &mut Vec<EsHit>,
+ max_score: &mut Option<f64>,
+ ) -> Result<Value, RerankError> {
+ let started = Instant::now();
+ let judge = local_judge_for(decide);
+
+ // The judge reads the hit exactly as the response carries it — the
+ // same builder the hosted stage uses, so the two stages can never
+ // drift on what text a hit carries.
+ let all: Vec<Candidate> = hits
+ .iter()
+ .enumerate()
+ .map(|(i, h)| {
+ crate::rerank_stage::unrestricted_candidate(
+ i,
+ h,
+ xerj_rerank::DEFAULT_MAX_DOC_CHARS,
+ )
+ })
+ .collect();
+ // A hit with no prose at all is not judged: the verdict on nothing
+ // means nothing, and the lexical arm would score it a meaningless 0.0
+ // next to a real miss that EARNED its 0.0.
+ let sendable: Vec<Candidate> = all
+ .iter()
+ .filter(|c| c.title.is_some() || !c.text.trim().is_empty())
+ .cloned()
+ .collect();
+
+ if !hits.is_empty() && sendable.is_empty() {
+ return Err(RerankError::Config(
+ "nothing to judge: none of the hits carries a string field in the response. \
+ The judge only sees what the response returns — make sure `_source` (or \
+ `fields`) returns the text fields"
+ .into(),
+ ));
+ }
+
+ let scores = judge.judge(&self.query, &sendable).await?;
+ let judged = scores.len();
+
+ // Judged probabilities by the caller's ordinal. One verdict per hit:
+ // `sendable` holds each ordinal once, and the judge contract returns
+ // one `Scored` per candidate in input order.
+ let mut by_ordinal: Vec<Option<f64>> = vec![None; hits.len()];
+ for s in &scores {
+ if let Some(slot) = by_ordinal.get_mut(s.ordinal) {
+ *slot = Some(s.score);
+ }
+ }
+
+ // Judged hits first, probability descending. `sort_by` is stable, so
+ // equal probabilities keep the engine's relative order — the judge
+ // breaks ties toward what the first stage said.
+ let mut ordered: Vec<(usize, f64)> = by_ordinal
+ .iter()
+ .enumerate()
+ .filter_map(|(i, p)| p.map(|p| (i, p)))
+ .collect();
+ ordered.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
+
+ let mut slots: Vec<Option<EsHit>> = std::mem::take(hits).into_iter().map(Some).collect();
+ let mut out = Vec::with_capacity(slots.len());
+ let mut kept = 0usize;
+ let mut dropped = 0usize;
+ let mut unjudged = 0usize;
+ for (ordinal, p) in ordered {
+ // Take the slot FIRST. A `continue` before the take would leave
+ // the hit in its slot, and the sweep over the leftovers below
+ // would count the same dropped hit twice.
+ let Some(mut h) = slots.get_mut(ordinal).and_then(Option::take) else {
+ continue;
+ };
+ if self.cfg.min_p.is_some_and(|min| p < min) {
+ dropped += 1;
+ continue;
+ }
+ // The engine's `_score` stays; the probability is its own field.
+ h.p_relevant = Some(p);
+ kept += 1;
+ out.push(h);
+ }
+ // What is left in the slots is the hits the judge never scored (no
+ // text at all). With a threshold they cannot be shown to clear it, so
+ // they are dropped and counted; without one they follow in the
+ // engine's order, marked unjudged by the absence of `_p_relevant`.
+ for h in slots.into_iter().flatten() {
+ if self.cfg.min_p.is_some() {
+ dropped += 1;
+ } else {
+ unjudged += 1;
+ out.push(h);
+ }
+ }
+ *hits = out;
+
+ // `max_score` describes the emitted page's `_score`, which the judge
+ // did not touch — but a `min_p` drop may have removed the hit that
+ // carried the maximum, so it is recomputed over what is left.
+ *max_score = hits
+ .iter()
+ .filter_map(|h| h.score)
+ .fold(None, |acc, s| match acc {
+ Some(m) if m >= s => Some(m),
+ _ => Some(s),
+ });
+
+ let mut info = json!({
+ "applied": true,
+ "scorer": judge.name(),
+ "query": self.query,
+ "judged": judged,
+ "kept": kept,
+ "dropped": dropped,
+ "took_ms": started.elapsed().as_millis() as u64,
+ });
+ // Present only when the caller set one: `min_p` is the pruning
+ // threshold, absent means "judge and reorder, drop nothing".
+ if let Some(min_p) = self.cfg.min_p {
+ info["min_p"] = json!(min_p);
+ }
+ // Present only when something was left unjudged.
+ if unjudged > 0 {
+ info["unjudged"] = json!(unjudged);
+ }
+ // The model arm only: never name a "model" that is arithmetic.
+ if let Some(model) = judge.model_id() {
+ info["model"] = json!(model);
+ }
+ Ok(info)
+ }
+}
+
+/// Which local judge this node runs, from the decide settings.
+///
+/// A node with the local decide tier armed judges with that head — the SAME
+/// loaded copy the decide ladder uses (the handle shares the process-wide
+/// model registry), one model in memory, two questions asked of it. A node
+/// without one (mode `history`, feature not compiled in, no model directory)
+/// judges with the deterministic lexical scorer, and the response says so in
+/// `judged.scorer`. That is a named fallback, not a degrade: nothing failed,
+/// and no caller was told a model judged when arithmetic did.
+fn local_judge_for(decide: &DecideSettings) -> LocalJudge {
+ #[cfg(feature = "decide-local")]
+ {
+ if let Some(tier) = decide.local_tier() {
+ return LocalJudge::Model(ModelJudge::from_model_dir(tier.model_dir().to_path_buf()));
+ }
+ LocalJudge::Lexical
+ }
+ #[cfg(not(feature = "decide-local"))]
+ {
+ let _ = decide;
+ LocalJudge::Lexical
+ }
+}
+
+/// Whether a raw search body carries a `judge` block that means something.
+///
+/// `"judge": null` is treated as absent everywhere — the same rule as
+/// `rerank` (see [`crate::rerank_stage::carries_rerank`]). One rule.
+pub fn carries_judge(body: &Value) -> bool {
+ body.get("judge").is_some_and(|j| !j.is_null())
+}
+
+/// Whether a raw search body carries EITHER second-stage block (`rerank` or
+/// `judge`). For the surfaces that run neither and refuse both — `_msearch`,
+/// the search templates, `_rank_eval` — so a body carrying only `judge` does
+/// not get a refusal that names `rerank`.
+pub fn carries_second_stage(body: &Value) -> bool {
+ crate::rerank_stage::carries_rerank(body) || carries_judge(body)
+}
+
+/// HTTP status for a judge fault that must reach the caller.
+pub fn status_for(e: &RerankError) -> u16 {
+ match e {
+ RerankError::Config(_) => 400,
+ // The judge's local model could not load or could not score the page:
+ // server state, not the caller's request and not a gateway fault —
+ // the same class the rerank stage gives its own unconfigured-server
+ // cases.
+ RerankError::LocalModel(_) => 503,
+ other => crate::rerank_stage::status_for(other),
+ }
+}
+
+/// ES-shaped error `type` for a judge fault: a request the caller can fix is
+/// an `illegal_argument_exception` like every other 400; the rest are XERJ's,
+/// and this stage's — not `rerank_exception`, which would name the stage that
+/// did not run.
+pub fn error_type_for(e: &RerankError) -> &'static str {
+ match e {
+ RerankError::Config(_) => "illegal_argument_exception",
+ RerankError::LocalModel(_) => "judge_exception",
+ other => crate::rerank_stage::error_type_for(other),
+ }
+}
+
+/// Why an endpoint that does not run the judge stage refuses the block. One
+/// sentence, shared by every surface, so they cannot tell a caller three
+/// different things.
+pub fn unsupported_reason(endpoint: &str) -> String {
+ format!(
+ "`judge` is not supported on {endpoint}: only `POST /{{index}}/_search` runs the judge \
+ stage. Send this search there, or remove the `judge` block — it is refused here \
+ rather than silently ignored"
+ )
+}
+
+/// The 400 for an endpoint that does not run the judge stage.
+///
+/// Same rule as the rerank stage's: parsers here ignore keys they do not
+/// know, so without this a `judge` block vanished without a word and the
+/// caller got unjudged, unpruned hits back under a 200.
+pub fn unsupported_on(endpoint: &str) -> Value {
+ let reason = unsupported_reason(endpoint);
+ json!({
+ "error": {
+ "root_cause": [{ "type": "illegal_argument_exception", "reason": reason }],
+ "type": "illegal_argument_exception",
+ "reason": reason,
+ },
+ "status": 400,
+ })
+}
+
+/// The 400 for a surface that refuses BOTH second stages, naming the block
+/// the body actually carries: `_msearch` with a `judge` block must not tell
+/// the caller about `rerank`.
+pub fn unsupported_second_stage_on(endpoint: &str, body: &Value) -> Value {
+ if crate::rerank_stage::carries_rerank(body) {
+ crate::rerank_stage::unsupported_on(endpoint)
+ } else {
+ unsupported_on(endpoint)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn search_body(judge: Value) -> EsSearchBody {
+ let mut body: EsSearchBody = serde_json::from_value(json!({
+ "query": {"match": {"body": "vitamin d"}},
+ "size": 10,
+ "judge": judge,
+ }))
+ .expect("a valid body");
+ body.judge = Some(judge);
+ body
+ }
+
+ #[test]
+ fn no_judge_block_is_inert() {
+ let mut body: EsSearchBody =
+ serde_json::from_value(json!({"query": {"match_all": {}}})).unwrap();
+ assert!(JudgePlan::prepare(&mut body, false).unwrap().is_none());
+ }
+
+ #[test]
+ fn null_judge_block_is_inert() {
+ let mut body: EsSearchBody =
+ serde_json::from_value(json!({"query": {"match_all": {}}, "judge": null})).unwrap();
+ assert!(JudgePlan::prepare(&mut body, false).unwrap().is_none());
+ }
+
+ #[test]
+ fn parses_min_p_and_defers_the_query_to_inference() {
+ let mut body = search_body(json!({"local": true, "min_p": 0.5}));
+ body.judge = Some(json!({"local": true, "min_p": 0.5}));
+ body.query = Some(json!({"match": {"body": "vitamin d"}}));
+ let plan = JudgePlan::prepare(&mut body, false)
+ .expect("valid")
+ .expect("a plan");
+ assert_eq!(plan.query, "vitamin d");
+ }
+
+ #[test]
+ fn refuses_the_shapes_it_cannot_judge_coherently() {
+ let scrolling = {
+ let mut b = search_body(json!({}));
+ b.judge = Some(json!({}));
+ JudgePlan::prepare(&mut b, true)
+ };
+ assert!(scrolling.unwrap_err().contains("scroll"));
+
+ let mut b = search_body(json!({}));
+ b.sort = Some(json!("_doc"));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("`sort`"), "{err}");
+
+ let mut b = search_body(json!({}));
+ b.search_after = Some(json!([1]));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("`search_after`"), "{err}");
+
+ let mut b = search_body(json!({}));
+ b.collapse = Some(json!({"field": "title"}));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("`collapse`"), "{err}");
+
+ let mut b = search_body(json!({}));
+ b.size = 0;
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("`size: 0`"), "{err}");
+
+ // `_source: false` alone: nothing to judge.
+ let mut b = search_body(json!({}));
+ b.source = Some(json!(false));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("nothing"), "{err}");
+ // ...but beside a `fields` clause it is coherent ("judge on that").
+ b.fields = Some(json!(["body"]));
+ assert!(JudgePlan::prepare(&mut b, false).is_ok());
+ }
+
+ #[test]
+ fn a_question_is_required_when_the_query_has_no_single_one() {
+ let mut b = search_body(json!({}));
+ b.judge = Some(json!({}));
+ b.query = Some(json!({"bool": {"must": [{"match": {"a": "x"}}]}}));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("`judge.query` is required"), "{err}");
+ // Spelled out, the same body judges.
+ b.judge = Some(json!({"query": "the question"}));
+ assert!(JudgePlan::prepare(&mut b, false).is_ok());
+ }
+
+ #[test]
+ fn config_errors_from_the_parser_surface_as_prepare_errors() {
+ let mut b = search_body(json!({"local": false}));
+ b.judge = Some(json!({"local": false}));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("cannot be false"), "{err}");
+
+ let mut b = search_body(json!({"min_p": 1.5}));
+ b.judge = Some(json!({"min_p": 1.5}));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("between 0 and 1"), "{err}");
+
+ let mut b = search_body(json!({"windwo": 1}));
+ b.judge = Some(json!({"windwo": 1}));
+ let err = JudgePlan::prepare(&mut b, false).unwrap_err();
+ assert!(err.contains("unknown `judge` field `windwo`"), "{err}");
+ }
+
+ #[test]
+ fn fault_statuses() {
+ assert_eq!(status_for(&RerankError::Config("x".into())), 400);
+ assert_eq!(status_for(&RerankError::LocalModel("no dir".into())), 503);
+ assert_eq!(
+ error_type_for(&RerankError::Config("x".into())),
+ "illegal_argument_exception"
+ );
+ assert_eq!(
+ error_type_for(&RerankError::LocalModel("no dir".into())),
+ "judge_exception"
+ );
+ }
+
+ #[test]
+ fn second_stage_detection_and_refusals_name_the_block_carried() {
+ assert!(carries_judge(&json!({"judge": {}})));
+ assert!(!carries_judge(&json!({"judge": null})));
+ assert!(!carries_judge(&json!({})));
+ assert!(carries_second_stage(&json!({"rerank": {}})));
+ assert!(carries_second_stage(&json!({"judge": {}})));
+ assert!(!carries_second_stage(&json!({"size": 10})));
+
+ // A body carrying only `judge` gets the `judge` refusal; one carrying
+ // `rerank` keeps the rerank refusal.
+ let judge_only = unsupported_second_stage_on("_msearch", &json!({"judge": {}}));
+ assert!(judge_only["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .starts_with("`judge` is not supported on _msearch"));
+ let rerank_too = unsupported_second_stage_on("_msearch", &json!({"rerank": {}}));
+ assert!(rerank_too["error"]["reason"]
+ .as_str()
+ .unwrap()
+ .starts_with("`rerank` is not supported on _msearch"));
+ assert_eq!(unsupported_on("_search/scroll")["status"], 400);
+ assert!(unsupported_reason("the native search API").contains("`judge`"));
+ }
+
+ // ── apply ─────────────────────────────────────────────────────────────
+
+ fn hit(id: &str, score: f64, text: &str) -> EsHit {
+ serde_json::from_value(json!({
+ "_index": "i", "_id": id, "_score": score,
+ "_source": {"body": text},
+ "matched_queries": null
+ }))
+ .expect("an EsHit")
+ }
+
+ fn plan(min_p: Option<f64>) -> JudgePlan {
+ let cfg = JudgeConfig::from_json(&match min_p {
+ Some(p) => json!({"query": "vitamin", "min_p": p}),
+ None => json!({"query": "vitamin"}),
+ })
+ .expect("a valid block");
+ JudgePlan {
+ cfg,
+ query: "vitamin".into(),
+ }
+ }
+
+ fn ids(hits: &[EsHit]) -> Vec<String> {
+ hits.iter().map(|h| h.id.clone()).collect()
+ }
+
+ #[tokio::test]
+ async fn judges_reorders_adds_p_and_leaves_score_alone() {
+ // Engine order by BM25 says b, a, c. The lexical judge (query
+ // "vitamin", page-local IDF) says a — both query words repeated —
+ // then b — one occurrence — then c, which shares no query word and
+ // scores exactly 0.0.
+ let mut hits = vec![
+ hit("b", 5.0, "vitamin once appears here"),
+ hit("a", 4.0, "vitamin vitamin"),
+ hit("c", 3.0, "nothing shared at all"),
+ ];
+ let mut max_score = Some(5.0);
+ let info = plan(None)
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+
+ // Reordered by the judge.
+ assert_eq!(ids(&hits), ["a", "b", "c"]);
+ // `_score` untouched (the engine's own numbers), `_p_relevant` added
+ // on every judged hit.
+ assert_eq!(hits[0].score, Some(4.0));
+ assert_eq!(hits[1].score, Some(5.0));
+ let ps: Vec<f64> = hits.iter().map(|h| h.p_relevant.unwrap()).collect();
+ assert!(ps[0] > ps[1], "{ps:?}");
+ assert_eq!(ps[2], 0.0, "no query word in the doc is exactly 0.0");
+ // All three were judged, none dropped.
+ assert_eq!(info["judged"], 3);
+ assert_eq!(info["kept"], 3);
+ assert_eq!(info["dropped"], 0);
+ assert!(info["unjudged"].is_null());
+ assert_eq!(info["scorer"], "lexical");
+ assert!(info["model"].is_null(), "the lexical arm is not a model");
+ // max_score still describes the page's `_score` values.
+ assert_eq!(max_score, Some(5.0));
+ }
+
+ #[tokio::test]
+ async fn min_p_drops_and_counts_below_threshold_hits() {
+ let mut hits = vec![
+ hit("b", 5.0, "vitamin once appears here"),
+ hit("a", 4.0, "vitamin vitamin"),
+ hit("c", 3.0, "nothing shared at all"),
+ ];
+ let mut max_score = Some(5.0);
+ let info = plan(Some(0.1))
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+ // c scored 0.0 < 0.1: dropped, and counted; a and b cleared it.
+ assert_eq!(ids(&hits), ["a", "b"]);
+ assert!(hits.iter().all(|h| h.p_relevant.is_some()));
+ assert_eq!(info["judged"], 3);
+ assert_eq!(info["kept"], 2);
+ assert_eq!(info["dropped"], 1);
+ assert_eq!(info["min_p"], 0.1);
+ assert_eq!(max_score, Some(5.0));
+ }
+
+ #[tokio::test]
+ async fn a_drop_that_removes_the_max_score_carrier_recomputes_it() {
+ let mut hits = vec![
+ hit("c", 9.0, "nothing shared at all"),
+ hit("a", 4.0, "vitamin vitamin"),
+ ];
+ let mut max_score = Some(9.0);
+ let info = plan(Some(0.5))
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+ assert_eq!(ids(&hits), ["a"]);
+ assert_eq!(info["dropped"], 1);
+ // `max_score` described c's 9.0; c was dropped, so it describes the
+ // page that is actually emitted now.
+ assert_eq!(max_score, Some(4.0));
+ }
+
+ #[tokio::test]
+ async fn a_hit_with_no_text_is_kept_unjudged_or_dropped_with_a_threshold() {
+ let bare = serde_json::from_value(json!({
+ "_index": "i", "_id": "bare", "_score": 7.0, "matched_queries": null
+ }))
+ .expect("an EsHit");
+ let mut hits = vec![hit("a", 4.0, "vitamin vitamin"), bare];
+ let mut max_score = Some(7.0);
+ let info = plan(None)
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+ // No threshold: the bare hit follows in the engine's order, marked
+ // unjudged by the absence of `_p_relevant` (never a made-up 0.0).
+ assert_eq!(ids(&hits), ["a", "bare"]);
+ assert!(hits[1].p_relevant.is_none());
+ assert_eq!(info["judged"], 1);
+ assert_eq!(info["unjudged"], 1);
+ assert_eq!(info["dropped"], 0);
+
+ // With one, a hit the judge never scored cannot be shown to clear it.
+ let bare = serde_json::from_value(json!({
+ "_index": "i", "_id": "bare", "_score": 7.0, "matched_queries": null
+ }))
+ .expect("an EsHit");
+ let mut hits = vec![hit("a", 4.0, "vitamin vitamin"), bare];
+ let info = plan(Some(0.5))
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+ assert_eq!(ids(&hits), ["a"]);
+ assert_eq!(info["dropped"], 1);
+ assert_eq!(info["kept"], 1);
+ assert!(info["unjudged"].is_null());
+ }
+
+ #[tokio::test]
+ async fn a_page_with_no_text_at_all_is_refused_not_judged_blind() {
+ let mut hits = vec![serde_json::from_value(json!({
+ "_index": "i", "_id": "x", "_score": 1.0, "matched_queries": null
+ }))
+ .unwrap()];
+ let mut max_score = Some(1.0);
+ let err = plan(None)
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .unwrap_err();
+ assert!(err.to_string().contains("nothing to judge"), "{err}");
+ }
+
+ #[tokio::test]
+ async fn empty_pages_judge_to_empty() {
+ let mut hits: Vec<EsHit> = Vec::new();
+ let mut max_score = None;
+ let info = plan(None)
+ .apply(&DecideSettings::history_only(), &mut hits, &mut max_score)
+ .await
+ .expect("judged");
+ assert_eq!(info["judged"], 0);
+ assert_eq!(info["kept"], 0);
+ assert!(hits.is_empty());
+ }
+}
diff --git a/engine/crates/xerj-api/src/lib.rs b/engine/crates/xerj-api/src/lib.rs
index 822a87653..ae6e1cb44 100644
--- a/engine/crates/xerj-api/src/lib.rs
+++ b/engine/crates/xerj-api/src/lib.rs
@@ -73,6 +73,7 @@ pub mod es_compat;
pub mod extract;
pub mod graph_api;
pub mod ism_api;
+pub mod judge_stage;
pub mod memory_api;
pub mod native;
pub mod rerank_stage;
diff --git a/engine/crates/xerj-api/src/native.rs b/engine/crates/xerj-api/src/native.rs
index f6178012d..d32d6c79d 100644
--- a/engine/crates/xerj-api/src/native.rs
+++ b/engine/crates/xerj-api/src/native.rs
@@ -66,6 +66,10 @@ pub struct NativeSearchRequest {
/// engine's order back under a 200, believing it had been reranked.
#[serde(default)]
pub rerank: Option<Value>,
+ /// Same rule for the local judge's block (issue #1060): declared so it is
+ /// refused by name, never silently dropped.
+ #[serde(default)]
+ pub judge: Option<Value>,
}
fn default_size() -> usize {
@@ -497,11 +501,14 @@ pub async fn search(
// recorded once at completion via `record_query`.
let _search_guard = state.metrics.active_search_guard();
- if req.rerank.is_some() {
+ if req.rerank.is_some() || req.judge.is_some() {
crate::rerank_stage::record_refused(&state.metrics);
- let error = xerj_common::XerjError::invalid_query(crate::rerank_stage::unsupported_reason(
- "the native search API",
- ));
+ let reason = if req.rerank.is_some() {
+ crate::rerank_stage::unsupported_reason("the native search API")
+ } else {
+ crate::judge_stage::unsupported_reason("the native search API")
+ };
+ let error = xerj_common::XerjError::invalid_query(reason);
return native_error(
error,
Some(&request_id),
diff --git a/engine/crates/xerj-api/src/rerank_stage.rs b/engine/crates/xerj-api/src/rerank_stage.rs
index 8aa72ea2f..9700dbd26 100644
--- a/engine/crates/xerj-api/src/rerank_stage.rs
+++ b/engine/crates/xerj-api/src/rerank_stage.rs
@@ -86,8 +86,9 @@ pub struct RerankPlan {
///
/// Deliberately shallow. A `bool` tree, a `hybrid` with two different arms or a
/// `query_string` with operators has no single "question", and picking one
-/// would rank against a guess. Those callers pass `rerank.query`.
-fn infer_query(q: &Value) -> Option<String> {
+/// would rank against a guess. Those callers pass `rerank.query` (or, for the
+/// local judge, `judge.query`).
+pub(crate) fn infer_query(q: &Value) -> Option<String> {
let obj = q.as_object()?;
if obj.len() != 1 {
return None;
@@ -203,14 +204,17 @@ fn fields_requests(fields: Option<&Value>, field: &str) -> bool {
/// text that could never leave the node (measured in review). Stopping at the
/// budget here bounds every later copy by `window × 2 × max_doc_chars`
/// characters, and what is sent is character-for-character what it was.
-struct Prose {
+pub(crate) struct Prose {
buf: String,
/// Characters still allowed.
room: usize,
}
+// (Fields stay private; the constructor and the pushes below are the whole
+// interface, and the judge stage builds its candidates through the same
+// methods.)
impl Prose {
- fn new(max_chars: usize) -> Self {
+ pub(crate) fn new(max_chars: usize) -> Self {
Self {
buf: String::new(),
room: max_chars,
@@ -243,7 +247,7 @@ impl Prose {
/// Append every string under `v` — a string, or an array of strings.
/// Numbers, booleans, vectors and nested objects are not prose and are not
/// sent.
- fn push(&mut self, v: &Value) {
+ pub(crate) fn push(&mut self, v: &Value) {
match v {
Value::String(s) => self.push_str(s),
Value::Array(items) => items
@@ -260,7 +264,10 @@ impl Prose {
}
/// `a.b.c` looked up first as a literal key, then as a path.
-fn lookup<'a>(obj: &'a serde_json::Map<String, Value>, field: &str) -> Option<&'a Value> {
+pub(crate) fn lookup<'a>(
+ obj: &'a serde_json::Map<String, Value>,
+ field: &str,
+) -> Option<&'a Value> {
if let Some(v) = obj.get(field) {
return Some(v);
}
@@ -268,6 +275,62 @@ fn lookup<'a>(obj: &'a serde_json::Map<String, Value>, field: &str) -> Option<&'
lookup(obj.get(head)?.as_object()?, rest)
}
+/// The candidate a judge reads when no field list restricts what leaves the
+/// node: the title in its own slot, the matching passage FIRST in the text,
+/// then every string `_source` returns, then the hit's `fields` in name
+/// order.
+///
+/// Free-standing and `pub(crate)` so the local judge stage
+/// ([`crate::judge_stage`]) builds its candidates through the same code the
+/// hosted stage does: one implementation of "the judge sees what the response
+/// returns", so the two stages can never drift on what text a hit carries.
+pub(crate) fn unrestricted_candidate(
+ ordinal: usize,
+ hit: &EsHit,
+ max_doc_chars: usize,
+) -> Candidate {
+ let mut title = Prose::new(max_doc_chars);
+ RerankPlan::field_text(hit, "title", &mut title);
+
+ let mut text = Prose::new(max_doc_chars);
+ // The matching passage goes FIRST when the response carries one. Text is
+ // cut at `max_doc_chars`, so on a long document an appended passage would
+ // be the part that gets cut — and it is the part that says why this hit
+ // matched.
+ let passage = xerj_query::executor::PASSAGE_RESPONSE_FIELD;
+ RerankPlan::field_text(hit, passage, &mut text);
+ let source = hit.source.as_ref().and_then(Value::as_object);
+ if let Some(obj) = source {
+ obj.iter()
+ .filter(|(k, _)| k.as_str() != "title")
+ .for_each(|(_, v)| text.push(v));
+ }
+ // Then what the hit returns under `fields` — where `fields`,
+ // `docvalue_fields`, `stored_fields` and `script_fields` put their
+ // values, once each, in a stable (sorted) order so the same response
+ // always sends the same text. See `RerankPlan::candidate` for the full
+ // rationale.
+ if let Some(returned) = &hit.fields {
+ let mut names: Vec<&String> = returned
+ .keys()
+ .filter(|k| {
+ k.as_str() != "title"
+ && k.as_str() != passage
+ && !source.is_some_and(|o| o.contains_key(k.as_str()))
+ })
+ .collect();
+ names.sort();
+ for name in names {
+ text.push(&returned[name]);
+ }
+ }
+ Candidate {
+ ordinal,
+ title: (!title.is_empty()).then_some(title.buf),
+ text: text.buf,
+ }
+}
+
impl RerankPlan {
/// Validate the request and widen the page to the rerank window.
///
@@ -437,7 +500,11 @@ impl RerankPlan {
/// Collect the prose a hit returns under one field name: `_source` first,
/// then the hit's `fields` — which is how fetched-not-stored values get
/// judged.
- fn field_text(hit: &EsHit, name: &str, out: &mut Prose) {
+ ///
+ /// Free-standing and `pub(crate)` so the judge stage reads exactly the
+ /// same text the hosted provider would: one implementation of "the judge
+ /// sees what the response returns".
+ pub(crate) fn field_text(hit: &EsHit, name: &str, out: &mut Prose) {
let src = hit.source.as_ref().and_then(Value::as_object);
if let Some(v) = src
.and_then(|o| lookup(o, name))
@@ -460,68 +527,25 @@ impl RerankPlan {
}
fn candidate(&self, ordinal: usize, hit: &EsHit) -> Candidate {
+ // Unrestricted is the reading the local judge stage uses too — see
+ // [`unrestricted_candidate`].
+ let Some(fields) = &self.cfg.fields else {
+ return unrestricted_candidate(ordinal, hit, self.cfg.max_doc_chars);
+ };
// `rerank.fields` is an egress control, so it is exhaustive: a caller
// who wrote `["body"]` has said what may leave the machine, and the
- // title is not on that list. Unrestricted, the title is promoted to its
- // own slot because a judge reads it differently from body text.
- let title_allowed = match &self.cfg.fields {
- None => true,
- Some(fields) => fields.iter().any(|f| f == "title"),
- };
+ // title is not on that list.
// Title and text are each cut at `max_doc_chars` as they are
// collected — see [`Prose`].
+ let title_allowed = fields.iter().any(|f| f == "title");
let mut title = Prose::new(self.cfg.max_doc_chars);
if title_allowed {
Self::field_text(hit, "title", &mut title);
}
let mut text = Prose::new(self.cfg.max_doc_chars);
- match &self.cfg.fields {
- Some(fields) => {
- for f in fields.iter().filter(|f| f.as_str() != "title") {
- Self::field_text(hit, f, &mut text);
- }
- }
- None => {
- // The matching passage goes FIRST when the response carries
- // one. Text is cut at `max_doc_chars`, so on a long document an
- // appended passage would be the part that gets cut — and it is
- // the part that says why this hit matched.
- let passage = xerj_query::executor::PASSAGE_RESPONSE_FIELD;
- Self::field_text(hit, passage, &mut text);
- let source = hit.source.as_ref().and_then(Value::as_object);
- if let Some(obj) = source {
- obj.iter()
- .filter(|(k, _)| k.as_str() != "title")
- .for_each(|(_, v)| text.push(v));
- }
- // Then what the hit returns under `fields` — where `fields`,
- // `docvalue_fields`, `stored_fields` and `script_fields` put
- // their values. "The judge sees what the response returns"
- // includes them: the docs and `prepare`'s own refusal both say
- // "return the text through `fields`", and until this loop
- // existed `_source: false` + `fields: ["body"]` + `rerank: {}`
- // followed that advice into a second 400 ("nothing to judge").
- // A name `_source` already returned at the top level is
- // skipped, so a value present in both is sent once. `fields`
- // is a `HashMap`; the names are sorted so the same response
- // always sends the same text (the cut at `max_doc_chars` would
- // otherwise fall on a different field from call to call).
- if let Some(returned) = &hit.fields {
- let mut names: Vec<&String> = returned
- .keys()
- .filter(|k| {
- k.as_str() != "title"
- && k.as_str() != passage
- && !source.is_some_and(|o| o.contains_key(k.as_str()))
- })
- .collect();
- names.sort();
- for name in names {
- text.push(&returned[name]);
- }
- }
- }
+ for f in fields.iter().filter(|f| f.as_str() != "title") {
+ Self::field_text(hit, f, &mut text);
}
Candidate {
ordinal,
@@ -842,6 +866,10 @@ pub fn status_for(e: &RerankError) -> u16 {
// The server is not configured to reach the provider: not the
// caller's mistake, and not a gateway fault either.
RerankError::MissingKey(_) => 503,
+ // Originates from the judge stage (a local model that cannot load or
+ // score), never from the hosted provider — same class as MissingKey:
+ // server state, not the caller and not a gateway fault.
+ RerankError::LocalModel(_) => 503,
RerankError::Transport(_)
| RerankError::Status { .. }
| RerankError::Malformed(_)
diff --git a/engine/crates/xerj-api/src/responses.rs b/engine/crates/xerj-api/src/responses.rs
index f1137b84f..d84f21af2 100644
--- a/engine/crates/xerj-api/src/responses.rs
+++ b/engine/crates/xerj-api/src/responses.rs
@@ -340,6 +340,16 @@ pub struct EsHit {
/// original malformed values that triggered `_ignored`.
#[serde(skip_serializing_if = "Option::is_none")]
pub ignored_field_values: Option<Value>,
+ /// `_p_relevant` — XERJ's local-judge probability (issue #1060): the
+ /// judge's `0..=1` relevance probability for this hit, present only on
+ /// a search that carried a `judge` block (and only on hits the judge
+ /// scored). Unlike the hosted `rerank` stage — which REPLACES `_score`
+ /// with its probability — the judge leaves the engine's `_score`
+ /// untouched and adds this field, so a client sorting by `_score` keeps
+ /// the engine's own number and a client thresholding `_p_relevant` gets
+ /// an absolute scale `_score` never had.
+ #[serde(rename = "_p_relevant", skip_serializing_if = "Option::is_none")]
+ pub p_relevant: Option<f64>,
}
/// The `total` sub-object in `hits`.
diff --git a/engine/crates/xerj-api/tests/judge_stage_http.rs b/engine/crates/xerj-api/tests/judge_stage_http.rs
new file mode 100644
index 000000000..5d4a5b717
--- /dev/null
+++ b/engine/crates/xerj-api/tests/judge_stage_http.rs
@@ -0,0 +1,823 @@
+//! End-to-end contract checks for the local `judge` stage of `_search` (issue
+//! #1060), over HTTP against an in-process node.
+//!
+//! The judge needs no provider — that is its whole point — so most tests here
+//! run with reranking DISABLED (`state.rerank` replaced wholesale, as
+//! `rerank_stage_http.rs` does) and nothing ambient can be reached. The one
+//! composition test brings up the same System One stub the rerank suite uses,
+//! reduced to its overlap scoring, because the contract it pins is the ORDER
+//! the two stages run in when both blocks ride one request.
+//!
+//! The lexical judge is deterministic, so these tests pin exact behaviour —
+//! but they pin CONTRACT properties (presence, order, counts, refusals), not
+//! ranking quality: nothing here says anything about which documents a real
+//! model would judge relevant. That is the BEIR gate's job at release.
+
+use std::sync::{
+ atomic::{AtomicU64, Ordering},
+ Arc,
+};
+
+use axum::body::Body;
+use axum::extract::State;
+use axum::http::{HeaderMap, Request, StatusCode};
+use axum::response::IntoResponse;
+use axum::routing::post;
+use serde_json::{json, Value};
+use tower::ServiceExt;
+use xerj_rerank::ProviderSettings;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// A minimal System One stub — for the composition tests only
+// ─────────────────────────────────────────────────────────────────────────────
+
+fn words(s: &str) -> std::collections::HashSet<String> {
+ s.to_lowercase()
+ .split(|c: char| !c.is_alphanumeric())
+ .filter(|w| !w.is_empty())
+ .map(str::to_string)
+ .collect()
+}
+
+/// Scores by the fraction of the query's words found in the document — the
+/// same shape `rerank_stage_http.rs`'s stub uses. A test double, not a judge.
+///
+/// Every request increments `calls`: the zero-token claim of the judge stage
+/// is asserted against that counter, so the assertion observes real egress,
+/// not the absence of a crash.
+async fn systemone(
+ State(calls): State<Arc<AtomicU64>>,
+ headers: HeaderMap,
+ body: String,
+) -> axum::response::Response {
+ let _ = headers;
+ calls.fetch_add(1, Ordering::SeqCst);
+ let parsed: Value = serde_json::from_str(&body).unwrap_or(Value::Null);
+ let query = words(parsed["state"].as_str().unwrap_or_default());
+ let mut answers = serde_json::Map::new();
+ if let Some(qs) = parsed["questions"].as_object() {
+ for (key, q) in qs {
+ let doc = q["instructions"]["document"].as_str().unwrap_or_default();
+ let have = words(doc);
+ let p = query.intersection(&have).count() as f64 / query.len().max(1) as f64;
+ answers.insert(key.clone(), json!({"type": "noul", "noul": p}));
+ }
+ }
+ axum::Json(json!({
+ "model": parsed["model"],
+ "answers": answers,
+ "usage": {"input_tokens": 100, "output_tokens": 10},
+ }))
+ .into_response()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// The node
+// ─────────────────────────────────────────────────────────────────────────────
+
+struct Node {
+ app: axum::Router,
+ /// The XERJ-native REST API (`/v1/...`), over the SAME state as `app`.
+ native: axum::Router,
+ _dir: tempfile::TempDir,
+}
+
+async fn node_with(settings: ProviderSettings) -> Node {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let mut config = xerj_common::config::Config::default();
+ config.server.data_dir = dir.path().to_string_lossy().into_owned();
+ config.storage.wal_sync = xerj_common::config::WalSync::Async;
+ let metrics = xerj_common::metrics::Metrics::new().expect("metrics");
+ let engine = xerj_engine::Engine::new(config.clone()).expect("engine");
+ let mut state = xerj_api::state::AppState::new(config, engine, metrics);
+ // The seam, same as the rerank suite: whatever TYPESAFE_* the developer's
+ // shell holds was read into `state.rerank` by `AppState::new` and is
+ // replaced wholesale here. The judge stage never consults it; only the
+ // composition test arms it.
+ state.rerank = Arc::new(settings);
+ Node {
+ native: xerj_api::router::build_native_router(state.clone()),
+ app: xerj_api::router::build_es_compat_router(state),
+ _dir: dir,
+ }
+}
+
+/// A node with reranking disabled: the judge is the only second stage that
+/// can run, and no ambient provider key can change that.
+async fn node() -> Node {
+ node_with(ProviderSettings::resolve(false, "", "", None, None)).await
+}
+
+/// A node with a live overlap stub, for the composition tests. Returns the
+/// node plus the counter of requests the stub received — the observed egress.
+async fn node_with_stub() -> (Node, Arc<AtomicU64>) {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind an ephemeral port");
+ let addr = listener.local_addr().expect("local addr");
+ let endpoint = format!("http://{addr}/v1/systemone");
+ let calls = Arc::new(AtomicU64::new(0));
+ let app = axum::Router::new()
+ .route("/v1/systemone", post(systemone))
+ .with_state(calls.clone());
+ tokio::spawn(async move {
+ let _ = axum::serve(listener, app).await;
+ });
+ (
+ node_with(ProviderSettings::with_key_and_endpoint(
+ "test-key", &endpoint,
+ ))
+ .await,
+ calls,
+ )
+}
+
+impl Node {
+ async fn raw(&self, method: &str, path: &str, body: String) -> (StatusCode, String) {
+ let response = self
+ .app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body))
+ .expect("request"),
+ )
+ .await
+ .expect("response");
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .expect("body");
+ (status, String::from_utf8_lossy(&bytes).into_owned())
+ }
+
+ async fn call(&self, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let (status, text) = self.raw(method, path, body.to_string()).await;
+ (status, serde_json::from_str(&text).unwrap_or(Value::Null))
+ }
+
+ async fn search(&self, path: &str, body: Value) -> (StatusCode, Value) {
+ self.call("POST", path, body).await
+ }
+
+ /// The four documents the rerank suite seeds: doc 2 is the only one that
+ /// carries every word of Q, which is what makes the judge's job checkable.
+ async fn seed_kb(&self) {
+ let (st, b) = self
+ .call(
+ "PUT",
+ "/kb",
+ json!({"mappings": {"properties": {
+ "title": {"type": "text"},
+ "body": {"type": "text"},
+ "cat": {"type": "keyword"},
+ "n": {"type": "integer"}
+ }}}),
+ )
+ .await;
+ assert!(st.is_success(), "create kb: {st} {b}");
+ for (id, title, body, cat, n) in [
+ (
+ "1",
+ "Bone health basics",
+ "vitamin supplements are popular. vitamin vitamin vitamin vitamin.",
+ "health",
+ 1,
+ ),
+ (
+ "2",
+ "Trial results",
+ "vitamin d supplementation improved bone density in the treatment group",
+ "trial",
+ 2,
+ ),
+ (
+ "3",
+ "Cooking",
+ "vitamin rich vegetables for dinner",
+ "food",
+ 3,
+ ),
+ (
+ "4",
+ "Density of materials",
+ "bone china has high density",
+ "materials",
+ 4,
+ ),
+ ] {
+ let (st, b) = self
+ .call(
+ "PUT",
+ &format!("/kb/_doc/{id}"),
+ json!({"title": title, "body": body, "cat": cat, "n": n}),
+ )
+ .await;
+ assert!(st.is_success(), "index kb/{id}: {st} {b}");
+ }
+ let (st, b) = self.call("POST", "/kb/_refresh", json!({})).await;
+ assert!(st.is_success(), "refresh kb: {st} {b}");
+ }
+}
+
+const Q: &str = "vitamin d supplementation bone density";
+
+fn match_q() -> Value {
+ json!({"match": {"body": Q}})
+}
+
+fn ids(r: &Value) -> Vec<String> {
+ r["hits"]["hits"]
+ .as_array()
+ .map(|a| {
+ a.iter()
+ .map(|h| h["_id"].as_str().unwrap_or_default().to_string())
+ .collect()
+ })
+ .unwrap_or_default()
+}
+
+fn ps(r: &Value) -> Vec<f64> {
+ r["hits"]["hits"]
+ .as_array()
+ .map(|a| {
+ a.iter()
+ .map(|h| h["_p_relevant"].as_f64().unwrap_or(f64::NAN))
+ .collect()
+ })
+ .unwrap_or_default()
+}
+
+fn reason(r: &Value) -> String {
+ r["error"]["reason"]
+ .as_str()
+ .unwrap_or_default()
+ .to_string()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Opt-in: absent by default, shape unchanged
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn no_judge_block_leaves_the_response_untouched() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ for body in [
+ json!({"query": match_q(), "size": 4}),
+ // `"judge": null` is the same as no `judge` key everywhere.
+ json!({"query": match_q(), "size": 4, "judge": null}),
+ ] {
+ let (st, r) = node.search("/kb/_search", body).await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ assert!(r.get("judged").is_none(), "opt-in: {r}");
+ assert_eq!(ids(&r).len(), 4);
+ for h in r["hits"]["hits"].as_array().unwrap() {
+ assert!(
+ h.get("_p_relevant").is_none(),
+ "no judge, no probability on the wire: {h}"
+ );
+ assert!(
+ h["_score"].is_number(),
+ "and the engine's score stands alone: {h}"
+ );
+ }
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Applied: _p_relevant per hit, reorder, the judged block
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn judged_hits_carry_a_probability_and_the_block_reports_the_pass() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (st, text) = node
+ .raw(
+ "POST",
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {"local": true}}).to_string(),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{text}");
+ let r: Value = serde_json::from_str(&text).unwrap();
+
+ // Every hit was judged (all four carry text) and reordered by the
+ // probability, descending.
+ let probabilities = ps(&r);
+ assert_eq!(probabilities.len(), 4, "{r}");
+ assert!(
+ probabilities.windows(2).all(|w| w[0] >= w[1]),
+ "descending: {probabilities:?} in {:?}",
+ ids(&r)
+ );
+ // Doc 2 carries every word of the question — the one hit the lexical
+ // judge is certainst about sits first.
+ assert_eq!(ids(&r)[0], "2", "{r}");
+
+ let judged = &r["judged"];
+ assert_eq!(judged["applied"], true, "{judged}");
+ assert_eq!(judged["scorer"], "lexical", "{judged}");
+ assert_eq!(judged["judged"], 4, "{judged}");
+ assert_eq!(judged["kept"], 4, "{judged}");
+ assert_eq!(judged["dropped"], 0, "{judged}");
+ assert_eq!(judged["query"], Q, "the inferred question is reported back");
+ assert!(judged["took_ms"].is_u64(), "{judged}");
+ assert!(
+ judged["min_p"].is_null(),
+ "no threshold asked for, none reported: {judged}"
+ );
+ assert!(
+ judged["unjudged"].is_null(),
+ "everything was judged: {judged}"
+ );
+ // The lexical arm is arithmetic: naming a "model" beside it would be the
+ // silent fake this stage exists to avoid.
+ assert!(judged["model"].is_null(), "{judged}");
+
+ // `judged` precedes `hits` on the wire: a reader that truncates long
+ // output from the bottom must still see which pass it is holding.
+ let at = |needle: &str| text.find(needle).unwrap_or(usize::MAX);
+ assert!(at("\"judged\"") < at("\"hits\""), "{text}");
+}
+
+#[tokio::test]
+async fn the_engine_score_stands_beside_the_probability() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (_, base) = node
+ .search("/kb/_search", json!({"query": match_q(), "size": 4}))
+ .await;
+ let (_, judged) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {}}),
+ )
+ .await;
+
+ // Same hits, same `_score` per hit — the judge adds a scale, it does not
+ // replace the engine's (the one deliberate difference from `rerank`).
+ let mut base_scores: std::collections::HashMap<String, f64> = std::collections::HashMap::new();
+ for h in base["hits"]["hits"].as_array().unwrap() {
+ base_scores.insert(
+ h["_id"].as_str().unwrap().to_string(),
+ h["_score"].as_f64().unwrap(),
+ );
+ }
+ for h in judged["hits"]["hits"].as_array().unwrap() {
+ let id = h["_id"].as_str().unwrap();
+ assert_eq!(
+ h["_score"].as_f64(),
+ base_scores.get(id).copied(),
+ "`_score` is the engine's own, untouched: {h}"
+ );
+ assert!(h["_p_relevant"].is_number(), "{h}");
+ }
+ // `max_score` describes `_score`, not `_p_relevant`.
+ let emitted_max = judged["hits"]["hits"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter_map(|h| h["_score"].as_f64())
+ .fold(None::<f64>, |acc: Option<f64>, s| match acc {
+ Some(m) if m >= s => Some(m),
+ _ => Some(s),
+ });
+ assert_eq!(
+ judged["hits"]["max_score"].as_f64(),
+ emitted_max,
+ "{}",
+ judged
+ );
+}
+
+#[tokio::test]
+async fn an_explicit_judge_query_wins_over_inference() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({
+ "query": match_q(), "size": 4,
+ "judge": {"query": "bone density"}
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ assert_eq!(r["judged"]["query"], "bone density", "{r}");
+}
+
+#[tokio::test]
+async fn text_returned_through_fields_is_judgeable() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ // `_source: false` alone is refused; beside a `fields` clause it is the
+ // documented way to judge without returning the whole source.
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({
+ "query": match_q(), "size": 4,
+ "_source": false,
+ "fields": ["body"],
+ "judge": {}
+ }),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ assert_eq!(r["judged"]["applied"], true, "{r}");
+ assert_eq!(r["judged"]["judged"], 4, "{r}");
+ assert!(ps(&r).iter().all(|p| p.is_finite()), "{r}");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// min_p: drops are counted, the total stays the engine's
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn min_p_drops_below_threshold_hits_and_counts_them() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ // Read the page's probabilities first, then cut inside them: the cut is
+ // self-adjusting, so the test pins the CONTRACT (drop below, keep at or
+ // above, count both) rather than one scorer's arithmetic.
+ let (_, base) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {}}),
+ )
+ .await;
+ let base_ps = ps(&base);
+ assert!(base_ps.len() == 4, "{base}");
+ // A threshold at the third hit's probability keeps the top three (>=
+ // keeps it) and drops the rest.
+ let cut = base_ps[2];
+ assert!(cut > 0.0, "the fixture needs spread: {base_ps:?}");
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {"min_p": cut}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ let kept = ps(&r);
+ assert!(
+ kept.iter().all(|p| *p >= cut),
+ "nothing below the cut survives: {kept:?}"
+ );
+ assert_eq!(
+ kept.len(),
+ base_ps.iter().filter(|p| **p >= cut).count(),
+ "{r}"
+ );
+ assert_eq!(r["judged"]["judged"], 4, "{r}");
+ assert_eq!(r["judged"]["kept"], kept.len() as u64, "{r}");
+ assert_eq!(
+ r["judged"]["dropped"],
+ (4 - kept.len()) as u64,
+ "dropped is counted in the response: {r}"
+ );
+ assert_eq!(r["judged"]["min_p"], cut, "{r}");
+ // `hits.total` describes the full match set, not the pruned page.
+ assert_eq!(
+ r["hits"]["total"]["value"], base["hits"]["total"]["value"],
+ "{r}"
+ );
+}
+
+#[tokio::test]
+async fn a_threshold_of_one_drops_everything_and_says_so() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {"min_p": 1.0}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ // Lexical saturation approaches 1.0 only as tf grows without bound, so a
+ // finite page never clears a cut of exactly 1.0 — an honest, counted
+ // empty page rather than a silent unjudged one.
+ assert!(ids(&r).is_empty(), "{r}");
+ assert_eq!(r["judged"]["kept"], 0, "{r}");
+ assert_eq!(r["judged"]["dropped"], 4, "{r}");
+ assert!(r["hits"]["max_score"].is_null(), "{r}");
+}
+
+#[tokio::test]
+async fn a_threshold_of_zero_keeps_everything() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {"min_p": 0.0}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ assert_eq!(ids(&r).len(), 4, "{r}");
+ assert_eq!(r["judged"]["dropped"], 0, "{r}");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Refusals: 400, by name, before the search runs
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn every_refusal_is_a_400_that_names_the_problem() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let q = match_q();
+ let cases: Vec<(&str, &str, Value, &str)> = vec![
+ (
+ "judge not an object",
+ "/kb/_search",
+ json!({"query": q, "judge": "yes"}),
+ "must be an object",
+ ),
+ (
+ "sort",
+ "/kb/_search",
+ json!({"query": q, "sort": [{"n": "asc"}], "judge": {}}),
+ "sort",
+ ),
+ (
+ "?sort= in the URL",
+ "/kb/_search?sort=n:asc",
+ json!({"query": q, "judge": {}}),
+ "sort",
+ ),
+ (
+ "local false",
+ "/kb/_search",
+ json!({"query": q, "judge": {"local": false}}),
+ "cannot be false",
+ ),
+ (
+ "misspelt field",
+ "/kb/_search",
+ json!({"query": q, "judge": {"treshold": 0.5}}),
+ "treshold",
+ ),
+ (
+ "min_p outside 0..1",
+ "/kb/_search",
+ json!({"query": q, "judge": {"min_p": 7}}),
+ "between 0 and 1",
+ ),
+ (
+ "bool without judge.query",
+ "/kb/_search",
+ json!({"query": {"bool": {"must": [{"match": {"body": "vitamin"}}]}}, "judge": {}}),
+ "judge.query",
+ ),
+ (
+ "no query at all",
+ "/kb/_search",
+ json!({"judge": {}}),
+ "judge.query",
+ ),
+ (
+ "_source:false",
+ "/kb/_search",
+ json!({"query": q, "_source": false, "judge": {}}),
+ "_source",
+ ),
+ // ES's own search_after-without-sort validation (which runs before
+ // the stage's) fires first here — both are 400s, and the
+ // judge-specific refusal text is pinned by the unit tests on a body
+ // built directly. What this case pins over HTTP: the combination
+ // never 200s with unjudged hits.
+ (
+ "search_after",
+ "/kb/_search",
+ json!({"query": q, "search_after": [1], "judge": {}}),
+ "at least one field",
+ ),
+ (
+ "collapse",
+ "/kb/_search",
+ json!({"query": q, "collapse": {"field": "cat"}, "judge": {}}),
+ "collapse",
+ ),
+ (
+ "scroll",
+ "/kb/_search?scroll=1m",
+ json!({"query": q, "judge": {}}),
+ "scroll",
+ ),
+ (
+ "size 0",
+ "/kb/_search",
+ json!({"query": q, "size": 0, "judge": {}}),
+ "size: 0",
+ ),
+ ];
+ for (name, path, body, needle) in cases {
+ let (st, r) = node.search(path, body).await;
+ assert_eq!(st, StatusCode::BAD_REQUEST, "{name}: {r}");
+ // Matched against the whole error object: some refusals ride inside
+ // ES's "all shards failed" wrapper (root_cause / caused_by), and the
+ // contract is that the caller can find the named cause, not that it
+ // sits in one specific slot.
+ let why = r["error"].to_string();
+ assert!(why.contains(needle), "{name}: refusal must name it: {why}");
+ // The judge's own refusals are `illegal_argument_exception`s; the
+ // search_after case rides ES's `search_phase_execution_exception`
+ // wrapper because the shard-level validation fires first. Both are
+ // 400s that name the cause — that is the contract; the type taxonomy
+ // is the engine's own, already covered by its suites.
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Other surfaces refuse the block by name instead of dropping it
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn msearch_items_carrying_judge_are_refused_per_item() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let ndjson =
+ "{\"index\":\"kb\"}\n{\"query\":{\"match\":{\"body\":\"vitamin\"}},\"judge\":{}}\n";
+ let (st, text) = node.raw("POST", "/_msearch", ndjson.to_string()).await;
+ assert_eq!(
+ st,
+ StatusCode::OK,
+ "the rest of the batch still runs: {text}"
+ );
+ let r: Value = serde_json::from_str(&text).unwrap();
+ let items = r["responses"].as_array().unwrap();
+ assert_eq!(items.len(), 1, "{r}");
+ assert_eq!(items[0]["status"], 400, "{r}");
+ assert!(
+ reason(&items[0]).contains("`judge` is not supported on _msearch"),
+ "{r}"
+ );
+ // ...and a body carrying `rerank` keeps the rerank refusal, not the
+ // judge's.
+ let ndjson =
+ "{\"index\":\"kb\"}\n{\"query\":{\"match\":{\"body\":\"vitamin\"}},\"rerank\":{}}\n";
+ let (_, text) = node.raw("POST", "/_msearch", ndjson.to_string()).await;
+ let r: Value = serde_json::from_str(&text).unwrap();
+ assert!(
+ reason(&r["responses"][0]).contains("`rerank` is not supported on _msearch"),
+ "{r}"
+ );
+}
+
+#[tokio::test]
+async fn a_scroll_continuation_refuses_judge_instead_of_dropping_it() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let (st, r) = node
+ .search(
+ "/kb/_search?scroll=1m",
+ json!({"query": match_q(), "size": 2}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ let scroll_id = r["_scroll_id"].as_str().expect("scroll id").to_string();
+
+ let (st, r) = node
+ .search(
+ "/_search/scroll",
+ json!({"scroll": "1m", "scroll_id": scroll_id, "judge": {}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::BAD_REQUEST, "{r}");
+ assert!(reason(&r).contains("_search/scroll"), "{r}");
+}
+
+#[tokio::test]
+async fn the_native_search_api_refuses_judge_instead_of_dropping_it() {
+ let node = node().await;
+ node.seed_kb().await;
+
+ let send = |body: Value| {
+ let app = node.native.clone();
+ async move {
+ let response = app
+ .oneshot(
+ Request::builder()
+ .method("POST")
+ .uri("/v1/indices/kb/search")
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ (status, String::from_utf8_lossy(&bytes).into_owned())
+ }
+ };
+
+ // Its request struct ignores unknown keys, so without this a `judge`
+ // block vanished and the caller got unjudged hits under a 200.
+ let (st, text) = send(json!({"q": "vitamin", "judge": {}})).await;
+ assert_eq!(st, StatusCode::BAD_REQUEST, "{text}");
+ assert!(text.contains("`judge`"), "{text}");
+ assert!(text.contains("_search"), "says where to send it: {text}");
+
+ // Without the block the native search is untouched.
+ let (st, text) = send(json!({"q": "vitamin"})).await;
+ assert_eq!(st, StatusCode::OK, "{text}");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Composition with the hosted stage
+// ─────────────────────────────────────────────────────────────────────────────
+
+#[tokio::test]
+async fn judge_runs_after_the_hosted_rerank_on_the_same_search() {
+ let (node, calls) = node_with_stub().await;
+ node.seed_kb().await;
+
+ let (st, text) = node
+ .raw(
+ "POST",
+ "/kb/_search",
+ json!({
+ "query": match_q(), "size": 4,
+ "rerank": {},
+ "judge": {"min_p": 0.1}
+ })
+ .to_string(),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{text}");
+ let r: Value = serde_json::from_str(&text).unwrap();
+
+ // Both blocks are on the wire, in the order the stages ran: hosted first
+ // (it widened and reordered its window), local judge second (it re-judged
+ // and pruned the emitted page).
+ assert_eq!(r["_rerank"]["applied"], true, "{r}");
+ assert_eq!(r["judged"]["applied"], true, "{r}");
+ let at = |needle: &str| text.find(needle).unwrap_or(usize::MAX);
+ assert!(
+ at("\"_rerank\"") < at("\"judged\""),
+ "wire order is stage order: {text}"
+ );
+
+ // The hosted stage replaced `_score` with its probability; the judge left
+ // that alone and added `_p_relevant`. Two scales, both named.
+ for h in r["hits"]["hits"].as_array().unwrap() {
+ assert!(h["_score"].is_number(), "the provider's verdict: {h}");
+ assert!(h["_p_relevant"].is_number(), "the judge's verdict: {h}");
+ }
+ // And the judge's prune counted against its own pass.
+ assert_eq!(
+ r["judged"]["judged"].as_u64().unwrap(),
+ r["judged"]["kept"].as_u64().unwrap() + r["judged"]["dropped"].as_u64().unwrap(),
+ "kept + dropped accounting holds: {r}"
+ );
+ // Positive control for the zero-token test below: the hosted stage DID
+ // leave the node, the stub answered.
+ assert!(calls.load(Ordering::SeqCst) >= 1, "the stub was called");
+}
+
+#[tokio::test]
+async fn a_judged_search_never_contacts_the_hosted_provider() {
+ let (node, calls) = node_with_stub().await;
+ node.seed_kb().await;
+
+ // The stub is running and reachable, the node holds a live key and
+ // endpoint for it — and a judged search must not touch it. Zero egress is
+ // the stage's whole claim; it is asserted against the stub's own request
+ // counter, not inferred from the search succeeding.
+ let (st, r) = node
+ .search(
+ "/kb/_search",
+ json!({"query": match_q(), "size": 4, "judge": {}}),
+ )
+ .await;
+ assert_eq!(st, StatusCode::OK, "{r}");
+ assert_eq!(r["judged"]["scorer"], "lexical", "{r}");
+ assert_eq!(r["judged"]["judged"], 4, "{r}");
+ assert_eq!(
+ calls.load(Ordering::SeqCst),
+ 0,
+ "the judge is local: the hosted endpoint must not be called"
+ );
+}
diff --git a/engine/crates/xerj-mcp/src/lib.rs b/engine/crates/xerj-mcp/src/lib.rs
index 4b580f71b..05ac807c3 100644
--- a/engine/crates/xerj-mcp/src/lib.rs
+++ b/engine/crates/xerj-mcp/src/lib.rs
@@ -526,6 +526,52 @@ fn rerank_arg_schema(query_required: bool) -> Value {
json!({ "type": ["object", "boolean"], "description": description })
}
+/// JSON-schema fragment for the optional `judge` argument (issue #1060),
+/// shared by `xerj_search` and `xerj_hybrid_search` like [`rerank_arg_schema`].
+///
+/// Written for the agent deciding between the two second stages. The one
+/// thing it must never do is blur them: `rerank` is a paid third-party call
+/// that sends text off the node; `judge` is local, free, and sends nothing
+/// anywhere — a different tool for a different budget, with a different
+/// (lower, unproven) quality ceiling, and the description says both.
+fn judge_arg_schema(query_required: bool) -> Value {
+ let question = if query_required {
+ "`query` (string, REQUIRED for this tool): the natural-language question to \
+ judge documents against."
+ } else {
+ "`query` (string): the natural-language question to judge documents against. \
+ Filled in for you when the tool's `query` is a plain string; REQUIRED when \
+ the tool's `query` is a `bool`, a `knn`, or anything else with no single \
+ question in it (the node refuses rather than guess)."
+ };
+ let description = format!(
+ "Optional LOCAL judging of the returned hits — the zero-token second stage. \
+ The node re-scores every hit it was about to return, in process, against the \
+ question: no provider, no API key, no network call, and NO document text \
+ leaves the machine (unlike `rerank`). Each judged hit carries `_p_relevant`, \
+ a 0..1 relevance probability on an absolute scale — the engine's `_score` \
+ stays untouched beside it — and hits are reordered by it. \
+ `min_p` (0..1) drops hits judged below the probability and counts them in the \
+ response's `judged` block (`{{kept, dropped}}`): an absolute relevance cut you \
+ can threshold, which a BM25 score cannot give you. \
+ USE IT WHEN: you want a per-hit relevance number or a keep/drop cut on every \
+ search without cost, or the data must stay on the node. \
+ HONEST CEILING: with no local model armed the judge is a deterministic lexical \
+ scorer — it re-reads the same words the first stage read, with page-local \
+ statistics, and has NO semantic signal. The response's `judged.scorer` says \
+ which judge judged (`lexical` or `model`); read it before treating `_p_relevant` \
+ as understanding. Prefer `rerank` (paid, egress) when semantic judgement is \
+ worth the cost. \
+ Pass true or {{}} for defaults, or an object with: {question} \
+ `min_p` (0..1): drop hits judged below this probability. \
+ Cannot be combined with `sort`. \
+ READ THE RESPONSE'S `judged` BLOCK: `scorer` names the judge, `kept`/`dropped` \
+ count the page, `unjudged` counts hits with no text to judge (they carry no \
+ `_p_relevant` and sort last).",
+ );
+ json!({ "type": ["object", "boolean"], "description": description })
+}
+
/// JSON-schema fragment for the optional `max_tokens` argument, shared by all
/// five search tools (`xerj_search`, `xerj_hybrid_search`,
/// `xerj_semantic_search`, `xerj_vector_search`, `xerj_code_search`) so the
@@ -579,7 +625,10 @@ pub fn tool_specs() -> Value {
Optional `rerank` adds a second stage that re-judges the top hits with \
an external relevance model — read that argument's description before \
using it: it needs a provider the operator configured, and it sends \
- document text off the machine. \
+ document text off the machine. Optional `judge` re-judges the returned \
+ hits LOCALLY (zero tokens, nothing leaves the node) and adds a \
+ `_p_relevant` probability per hit — see that argument's description \
+ for its honest ceiling. \
Pass `max_tokens` to cap the response at a token budget (never exceeded; \
see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
@@ -600,6 +649,7 @@ pub fn tool_specs() -> Value {
"sort": { "description": "ES sort clause (array or object)." },
"_source": { "description": "Source filtering (bool, field, or {includes,excludes})." },
"rerank": rerank_arg_schema(false),
+ "judge": judge_arg_schema(false),
"max_tokens": max_tokens_arg_schema()
}
}
@@ -666,6 +716,9 @@ pub fn tool_specs() -> Value {
Optional `rerank` re-judges the fused top hits with an external \
relevance model; here `rerank.query` is REQUIRED, because a hybrid \
search has several sub-queries and no single question to read. \
+ Optional `judge` re-judges the fused hits LOCALLY (zero tokens, \
+ nothing leaves the node); `judge.query` is REQUIRED for the same \
+ reason. \
Pass `max_tokens` to cap the response at a token budget (never exceeded; \
see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
@@ -693,6 +746,7 @@ pub fn tool_specs() -> Value {
},
"size": { "type": "integer", "description": "Max fused hits to return." },
"rerank": rerank_arg_schema(true),
+ "judge": judge_arg_schema(true),
"max_tokens": max_tokens_arg_schema()
},
"required": ["index", "queries"]
@@ -1223,6 +1277,67 @@ fn apply_rerank(
Ok(())
}
+/// Normalise the optional `judge` argument into the `_search` body (issue
+/// #1060) — the local, zero-token counterpart of [`apply_rerank`].
+///
+/// Same shape, same `opt_typed` policy, same plain-question carry: the string
+/// path expands into a `bool` tree the node refuses to read a question from,
+/// so the caller's words are carried into `judge.query` or every plain-string
+/// judged search would be a 400. Refusals mirror the node's, using the node's
+/// own validator ([`xerj_rerank::judge::JudgeConfig::from_json`]), so an agent
+/// is told about a misspelt key without a round trip.
+fn apply_judge(
+ args: &Value,
+ body: &mut serde_json::Map<String, Value>,
+ plain_query: Option<&str>,
+ query_required: bool,
+) -> Result<(), String> {
+ let mut block = match args.get("judge") {
+ None | Some(Value::Null) | Some(Value::Bool(false)) => return Ok(()),
+ Some(Value::Bool(true)) => serde_json::Map::new(),
+ Some(Value::Object(o)) => o.clone(),
+ Some(_) => {
+ return Err(
+ "`judge` must be true, or an object such as {\"min_p\": 0.5} \
+ (pass {} for defaults)"
+ .to_string(),
+ )
+ }
+ };
+ if body.contains_key("sort") {
+ return Err(
+ "`judge` cannot be combined with `sort`: an explicit sort already fixes the \
+ order, and judging would reorder around it. Drop one of them"
+ .to_string(),
+ );
+ }
+ if !block.contains_key("query") {
+ match plain_query {
+ Some(q) => {
+ block.insert("query".into(), json!(q));
+ }
+ None if query_required => {
+ return Err(
+ "`judge.query` is required for a hybrid search: it fuses several \
+ sub-queries, so there is no single question to judge documents \
+ against. Pass the natural-language question, e.g. \
+ {\"judge\": {\"query\": \"why did the vpn drop?\"}}"
+ .to_string(),
+ )
+ }
+ // A DSL object: the node reads the question from `match`,
+ // `match_phrase`, `multi_match`, `semantic` and
+ // `simple_query_string`, and refuses by name for anything else.
+ // Left to it, so there is one implementation of that rule.
+ None => {}
+ }
+ }
+ let block = Value::Object(block);
+ xerj_rerank::judge::JudgeConfig::from_json(&block).map_err(|e| e.to_string())?;
+ body.insert("judge".into(), block);
+ Ok(())
+}
+
fn build_search(args: &Value) -> Result<BuiltRequest, String> {
// #962: `index` used to be required, but nothing an agent reads said what
// to pass — the first real tool call after `xerj autoindex` was an error
@@ -1273,6 +1388,7 @@ fn build_search(args: &Value) -> Result<BuiltRequest, String> {
_ => None,
};
apply_rerank(args, &mut body, plain_query, false)?;
+ apply_judge(args, &mut body, plain_query, false)?;
// String-mode defaults, only where the caller left them unset: project
// `_source` down to the citation fields and ask for `_passage` so the
// response is the matching snippet, not the whole file body (measured ~9x
@@ -1361,6 +1477,7 @@ fn build_hybrid(args: &Value) -> Result<BuiltRequest, String> {
body.insert("query".into(), json!({ "hybrid": Value::Object(hybrid) }));
copy_opt(args, &mut body, "size");
apply_rerank(args, &mut body, None, true)?;
+ apply_judge(args, &mut body, None, true)?;
Ok((
Method::Post,
format!("/{index}/_search"),
@@ -2074,6 +2191,141 @@ mod tests {
}
}
+ // ── `judge` argument (issue #1060) ─────────────────────────────────────
+
+ #[test]
+ fn judge_is_absent_unless_asked_for() {
+ for args in [
+ json!({ "index": "d", "query": "how do refunds work" }),
+ json!({ "index": "d", "query": "how do refunds work", "judge": null }),
+ json!({ "index": "d", "query": "how do refunds work", "judge": false }),
+ ] {
+ let (_, _, body) = built(build_search(&args));
+ assert!(
+ body.get("judge").is_none(),
+ "no `judge` block means no judging and an unchanged response: {body}"
+ );
+ }
+ }
+
+ #[test]
+ fn a_plain_string_query_becomes_the_judge_question() {
+ // Same defect class as `rerank`: the string path expands to a `bool`,
+ // which the node refuses to read a question from.
+ let (_, _, body) = built(build_search(&json!({
+ "index": "d", "query": "how do refunds work", "judge": true
+ })));
+ assert!(body["query"]["bool"].is_object(), "still definition-first");
+ assert_eq!(body["judge"]["query"], "how do refunds work");
+
+ let (_, _, body) = built(build_search(&json!({
+ "index": "d", "query": "how do refunds work", "judge": { "min_p": 0.5 }
+ })));
+ assert_eq!(body["judge"]["query"], "how do refunds work");
+ assert_eq!(body["judge"]["min_p"], 0.5);
+ }
+
+ #[test]
+ fn an_explicit_judge_query_is_never_overwritten() {
+ let (_, _, body) = built(build_search(&json!({
+ "index": "d", "query": "refund",
+ "judge": { "query": "What is the refund window for annual plans?" }
+ })));
+ assert_eq!(
+ body["judge"]["query"],
+ "What is the refund window for annual plans?"
+ );
+ }
+
+ #[test]
+ fn judge_and_rerank_compose_on_the_same_search() {
+ // The node runs the hosted stage first, then judges the emitted page.
+ // Both blocks ride the body; neither may drop the other.
+ let (_, _, body) = built(build_search(&json!({
+ "index": "d", "query": "how do refunds work",
+ "rerank": { "window": 20 },
+ "judge": { "min_p": 0.4 }
+ })));
+ assert_eq!(body["rerank"]["window"], 20);
+ assert_eq!(body["judge"]["min_p"], 0.4);
+ assert_eq!(body["judge"]["query"], "how do refunds work");
+ }
+
+ #[test]
+ fn judge_refusals_happen_before_the_round_trip() {
+ let bad = |args: Value| build_search(&args).expect_err("must be refused");
+
+ let e = bad(json!({ "index": "d", "query": "q", "judge": "yes" }));
+ assert!(e.contains("must be true, or an object"), "{e}");
+
+ let e = bad(json!({ "index": "d", "query": "q", "sort": ["_doc"], "judge": true }));
+ assert!(e.contains("sort"), "{e}");
+
+ // The node's own validator: a misspelt key is named, not dropped.
+ let e = bad(json!({ "index": "d", "query": "q", "judge": { "treshold": 0.5 } }));
+ assert!(e.contains("treshold"), "{e}");
+
+ let e = bad(json!({ "index": "d", "query": "q", "judge": { "min_p": 7 } }));
+ assert!(e.contains("between 0 and 1"), "{e}");
+
+ let e = bad(json!({ "index": "d", "query": "q", "judge": { "local": false } }));
+ assert!(e.contains("cannot be false"), "{e}");
+ }
+
+ #[test]
+ fn hybrid_judge_requires_the_question() {
+ let queries = json!([{ "query": { "match": { "body": "vpn" } } }]);
+ let e = build_hybrid(&json!({ "index": "h", "queries": queries, "judge": true }))
+ .expect_err("a hybrid has no single question");
+ assert!(e.contains("judge.query"), "{e}");
+
+ let (_, path, body) = built(build_hybrid(&json!({
+ "index": "h", "queries": queries,
+ "judge": { "query": "why did the vpn drop?", "min_p": 0.3 }
+ })));
+ assert_eq!(path, "/h/_search");
+ assert_eq!(body["judge"]["query"], "why did the vpn drop?");
+ assert_eq!(body["judge"]["min_p"], 0.3);
+ assert!(body["query"]["hybrid"].is_object());
+ }
+
+ #[test]
+ fn the_judge_description_tells_an_agent_what_it_needs_to_decide() {
+ let specs = tool_specs();
+ for tool in ["xerj_search", "xerj_hybrid_search"] {
+ let spec = specs
+ .as_array()
+ .unwrap()
+ .iter()
+ .find(|t| t["name"] == tool)
+ .unwrap();
+ let d = spec["inputSchema"]["properties"]["judge"]["description"]
+ .as_str()
+ .unwrap_or_else(|| panic!("{tool} must describe `judge`"));
+ for needle in [
+ "zero-token",
+ "NO document text leaves the machine",
+ "_p_relevant",
+ "min_p",
+ "judged",
+ // The honest ceiling: the lexical arm is named, never passed
+ // off as a model.
+ "HONEST CEILING",
+ "judged.scorer",
+ "lexical",
+ ] {
+ assert!(d.contains(needle), "{tool}: description lacks `{needle}`");
+ }
+ assert!(
+ !spec["inputSchema"]["required"]
+ .as_array()
+ .map(|r| r.iter().any(|r| r == "judge"))
+ .unwrap_or(false),
+ "{tool}: `judge` is optional"
+ );
+ }
+ }
+
// ── `max_tokens` argument (#1058) ─────────────────────────────────────
#[test]
diff --git a/engine/crates/xerj-rerank/Cargo.toml b/engine/crates/xerj-rerank/Cargo.toml
index afac4f13c..825c872d9 100644
--- a/engine/crates/xerj-rerank/Cargo.toml
+++ b/engine/crates/xerj-rerank/Cargo.toml
@@ -14,5 +14,18 @@ thiserror.workspace = true
tokio.workspace = true
tracing.workspace = true
+# The local judge's semantic arm (issue #1060): the node's local decision
+# head — xerj-ai's candle NLI pair scorer — scored against a relevance
+# hypothesis. Optional so the default build of this leaf crate stays
+# dependency-light; the deterministic lexical arm needs none of it.
+xerj-ai = { path = "../xerj-ai", optional = true }
+
+[features]
+default = []
+# Wire the local judge to the candle decision head (same loader, same model
+# directory, same shared registry as `xerj-ai/decide-local`).
+judge-local = ["dep:xerj-ai", "xerj-ai/decide-local"]
+
[dev-dependencies]
tokio = { workspace = true }
+tempfile = { workspace = true }
diff --git a/engine/crates/xerj-rerank/src/judge.rs b/engine/crates/xerj-rerank/src/judge.rs
new file mode 100644
index 000000000..fd51419b8
--- /dev/null
+++ b/engine/crates/xerj-rerank/src/judge.rs
@@ -0,0 +1,744 @@
+//! The local `judge` of `_search` — second-stage relevance scoring that never
+//! leaves the node and never bills a token (issue #1060: "the zero-token
+//! rerank").
+//!
+//! [`crate::Provider`] is the operator-pays arm: a hosted cross-encoder, one
+//! outbound call per search, text off the node. This module is the other
+//! option — a scorer that runs in process over the hits the response was
+//! already about to return, so a caller can ask "how likely is THIS hit to
+//! be relevant, as a number I can threshold" without paying for it or
+//! exporting text. Each judged hit carries the probability as `_p_relevant`
+//! (the hosted stage REPLACES `_score` with its probability; the judge
+//! leaves the engine's `_score` alone and adds a field, so a client that
+//! sorts by `_score` keeps seeing the engine's own number), and hits below
+//! `judge.min_p` are dropped and counted in the response's `judged` block.
+//!
+//! # Honest scope: what the lexical scorer is, and is not
+//!
+//! The always-compiled arm, [`LocalJudge::Lexical`], is a DETERMINISTIC
+//! LEXICAL scorer — window-local BM25 saturation over the judged page, IDF
+//! computed from the page itself, mapped into `0.0..=1.0` by the maximum
+//! attainable saturation (the formula and its provenance are on
+//! [`lexical_scores`]). It is not a cross-encoder and has no semantic
+//! signal: it re-reads the same words the first stage read, with page-local
+//! statistics and an absolute scale the first stage does not offer. Whether
+//! that beats the engine's own order is an empirical question with a written
+//! bar — the BEIR gate in issue #1060 ("loses → does not ship": 0.699
+//! SciFact / 0.345 NFCorpus beyond run spread, FiQA ≥ 0.30, harness at
+//! `benchmarks/beir-hybrid`) — and NO quality claim is made here. What IS
+//! claimed, and unit-tested, is: deterministic, bounded, and microseconds
+//! for a top-30 page.
+//!
+//! [`LocalJudge::Model`] (cargo feature `judge-local`) is the semantic arm:
+//! the node's local decision head — the candle NLI pair scorer landed in
+//! `xerj-ai/src/decide.rs` for #1057 — scoring premise-=-document against a
+//! relevance hypothesis and its negation. It is feature-gated exactly like
+//! `decide-local` because it IS the same loader and the same model
+//! directory: a node that armed `XERJ_DECIDE_MODE=local` has a judge model,
+//! and a node without one uses the lexical arm and SAYS SO in the response's
+//! `judged.scorer`. Model quality with the real trained checkpoint is
+//! likewise unmeasured here: the gate binds at release, against the real
+//! model.
+//!
+//! # Failure policy
+//!
+//! The lexical arm cannot fail. The model arm surfaces its failures
+//! ([`RerankError::LocalModel`] → HTTP 503): a caller who asked for judged
+//! hits and silently got unjudged ones has been misled, which is the
+//! silent-fake class this crate refuses everywhere (see the crate header).
+//! There is deliberately no degrade-to-lexical on a model fault —
+//! substituting a different scorer than the one the response claims is the
+//! same lie in the other direction.
+
+use std::collections::HashMap;
+
+use serde_json::Value;
+
+use crate::{Candidate, RerankError, Scored};
+
+/// `judge.min_p` is a probability. Both ends are valid: `0.0` judges and
+/// reorders but drops nothing, `1.0` keeps only what the scorer scored at
+/// certainty.
+pub const MIN_P_RANGE: (f64, f64) = (0.0, 1.0);
+
+/// Ceiling on `judge.query`. The same string, under the same ceiling, as
+/// [`crate::MAX_QUERY_CHARS`]: it is not sent anywhere (the judge is
+// in-process), but the lexical scorer tokenizes it on every judged search
+/// and the model arm folds it into every pair it scores, so a bound the
+/// caller cannot choose still belongs to the server.
+pub const MAX_QUERY_CHARS: usize = crate::MAX_QUERY_CHARS;
+
+// Only the `judge-local` arm names a model directory.
+#[cfg(feature = "judge-local")]
+use std::path::PathBuf;
+
+/// Parsed `judge` block from a search body.
+///
+/// `local` is not a field: `true` is the only mode, so [`Self::from_json`]
+/// accepts it (explicitly or by omission) and refuses anything else by name.
+#[derive(Debug, Clone)]
+pub struct JudgeConfig {
+ /// The question to judge documents against. `None` defers to the search
+ /// query when it is a shape a single question can be read from — the
+ /// same inference rule the `rerank` stage uses.
+ pub query: Option<String>,
+ /// Drop hits scoring below this probability. `None` keeps all and only
+ /// reorders.
+ pub min_p: Option<f64>,
+}
+
+impl JudgeConfig {
+ /// Parse the `judge` object from a search body.
+ ///
+ /// Unknown keys are rejected rather than ignored, for the same reason
+ /// [`crate::RerankConfig::from_json`] rejects them: a caller who
+ /// misspells `min_p` and silently gets unpruned results has been
+ /// misled.
+ pub fn from_json(v: &Value) -> Result<Self, RerankError> {
+ let obj = v
+ .as_object()
+ .ok_or_else(|| RerankError::Config("`judge` must be an object".into()))?;
+
+ const KNOWN: &[&str] = &["local", "min_p", "query"];
+ for k in obj.keys() {
+ if !KNOWN.contains(&k.as_str()) {
+ // Named, never echoed whole — the same clip the `rerank`
+ // parser applies to an unknown key.
+ let shown = crate::clip(k, crate::MAX_ECHOED_NAME_CHARS);
+ let ellipsis = if shown.len() < k.len() { "…" } else { "" };
+ return Err(RerankError::Config(format!(
+ "unknown `judge` field `{shown}{ellipsis}`; supported: {}",
+ KNOWN.join(", ")
+ )));
+ }
+ }
+
+ if let Some(l) = obj.get("local") {
+ match l {
+ Value::Bool(true) => {}
+ Value::Bool(false) => {
+ return Err(RerankError::Config(
+ "`judge.local` cannot be false: the local judge is the only judge. \
+ Remove the `judge` block to search unjudged"
+ .into(),
+ ));
+ }
+ _ => {
+ return Err(RerankError::Config(
+ "`judge.local` must be true (the local judge is the only judge)".into(),
+ ));
+ }
+ }
+ }
+
+ let min_p = match obj.get("min_p") {
+ None => None,
+ Some(t) => {
+ let t = t
+ .as_f64()
+ .ok_or_else(|| RerankError::Config("`judge.min_p` must be a number".into()))?;
+ if !(MIN_P_RANGE.0..=MIN_P_RANGE.1).contains(&t) {
+ return Err(RerankError::Config(
+ "`judge.min_p` must be between 0 and 1 — it is a probability".into(),
+ ));
+ }
+ Some(t)
+ }
+ };
+
+ let query = match obj.get("query") {
+ None => None,
+ Some(q) => {
+ let q = q
+ .as_str()
+ .ok_or_else(|| RerankError::Config("`judge.query` must be a string".into()))?;
+ if crate::longer_than(q, MAX_QUERY_CHARS) {
+ return Err(RerankError::Config(format!(
+ "`judge.query` must be at most {MAX_QUERY_CHARS} characters (this one \
+ is {} bytes): the judge reads it against every hit in the page",
+ q.len()
+ )));
+ }
+ if q.trim().is_empty() {
+ return Err(RerankError::Config(
+ "`judge.query` must not be empty".into(),
+ ));
+ }
+ Some(q.to_string())
+ }
+ };
+
+ Ok(Self { query, min_p })
+ }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// The lexical scorer — the always-compiled arm
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// BM25 saturation parameters: the same values the engine's own first stage
+/// uses (`xerj-fts/src/bm25.rs`, `DEFAULT_K1` = 1.2, `b` = 0.75 — BM25's
+/// standard parameters, inherited from Lucene). Reused rather than re-chosen
+/// so the judge's saturation curve behaves like the ranking the caller
+/// already knows.
+const K1: f64 = 1.2;
+const B: f64 = 0.75;
+
+/// Lowercase word tokens: alphanumeric runs, in order, via `char` boundaries
+/// only (a byte split here would panic on multi-byte text — the defect that
+/// crash-looped `autoindex`, see [`crate::clip`]). The same split the
+/// `rerank` conformance tests use for word overlap.
+fn tokens(s: &str) -> impl Iterator<Item = String> + '_ {
+ s.split(|c: char| !c.is_alphanumeric())
+ .filter(|w| !w.is_empty())
+ .map(str::to_lowercase)
+}
+
+/// The deterministic lexical judge.
+///
+/// For each candidate it computes the share of the query's term weight the
+/// document saturates, where saturation is the BM25 TF-normalisation
+/// `tf·(k1+1) / (tf + k1·(1 − b + b·dl/avgdl))` and the weight is
+/// window-local IDF `ln(1 + (N − df + 0.5)/(df + 0.5))` computed over the
+/// judged page itself — the only corpus this scorer has, which is what
+/// "self-judged" means in the issue. The result is divided by the maximum
+/// attainable score (every query term fully saturated: `Σidf·(k1+1)`), which
+/// maps it into `0.0..=1.0` with a fixed meaning a caller can threshold:
+///
+/// * a document containing none of the query's words scores exactly `0.0`;
+/// * a short document saturated with all of them approaches `1.0`;
+/// * a term every document shares contributes little (common in the page),
+/// a term one document holds contributes its full IDF weight.
+///
+/// Deterministic by construction: a pure function of `(query, candidates)`
+/// with no clock, no randomness and no model. The same inputs always judge
+/// to the same probabilities, which is what makes `min_p` a stable
+/// threshold. Returns one [`Scored`] per candidate, in input order, with the
+/// caller's ordinals untouched.
+pub fn lexical_scores(query: &str, candidates: &[Candidate]) -> Vec<Scored> {
+ // Query terms, unique, in first-seen order — the order fixes the
+ // summation order, so f64 rounding is reproducible too.
+ let mut q_terms: Vec<String> = Vec::new();
+ {
+ let mut seen: std::collections::HashSet<String> = std::collections::HashSet::new();
+ for t in tokens(query) {
+ if seen.insert(t.clone()) {
+ q_terms.push(t);
+ }
+ }
+ }
+ if q_terms.is_empty() || candidates.is_empty() {
+ // No query words to judge by (or nothing to judge): every candidate
+ // scores 0.0, a score a `min_p` above 0 will drop and no `min_p`
+ // will reorder. Inventing spread here would be ranking by nothing.
+ return candidates
+ .iter()
+ .map(|c| Scored {
+ ordinal: c.ordinal,
+ score: 0.0,
+ })
+ .collect();
+ }
+
+ // Tokenize each candidate once: title and text are one bag of words —
+ // a judge reads a document, not a schema.
+ let docs: Vec<Vec<String>> = candidates
+ .iter()
+ .map(|c| {
+ let mut ts: Vec<String> = Vec::new();
+ if let Some(t) = &c.title {
+ ts.extend(tokens(t));
+ }
+ ts.extend(tokens(&c.text));
+ ts
+ })
+ .collect();
+
+ let n = docs.len() as f64;
+ let avgdl = {
+ let total: usize = docs.iter().map(Vec::len).sum();
+ (total as f64 / n).max(1.0)
+ };
+
+ // Window-local document frequency and IDF per query term, in q_terms
+ // order. df ≤ N always, so the ln argument is ≥ 1 and every weight is
+ // positive — a term no document holds simply saturates nowhere.
+ let idf: Vec<f64> = q_terms
+ .iter()
+ .map(|q| {
+ let df = docs.iter().filter(|d| d.iter().any(|t| t == q)).count() as f64;
+ (1.0 + (n - df + 0.5) / (df + 0.5)).ln()
+ })
+ .collect();
+ // The maximum attainable numerator: every query term at full saturation.
+ let max_score: f64 = idf.iter().sum::<f64>() * (K1 + 1.0);
+
+ let qset: std::collections::HashSet<&str> = q_terms.iter().map(String::as_str).collect();
+ candidates
+ .iter()
+ .enumerate()
+ .map(|(i, c)| {
+ let d = &docs[i];
+ let norm = K1 * (1.0 - B + B * d.len() as f64 / avgdl);
+ let mut tf: HashMap<&str, u64> = HashMap::new();
+ for t in d {
+ if qset.contains(t.as_str()) {
+ *tf.entry(t.as_str()).or_insert(0) += 1;
+ }
+ }
+ let numerator: f64 = idf
+ .iter()
+ .zip(q_terms.iter())
+ .map(|(w, q)| {
+ let tf = tf.get(q.as_str()).copied().unwrap_or(0) as f64;
+ w * (tf * (K1 + 1.0) / (tf + norm))
+ })
+ .sum();
+ Scored {
+ ordinal: c.ordinal,
+ score: (numerator / max_score).clamp(0.0, 1.0),
+ }
+ })
+ .collect()
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// The judge arms
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A local judge: scores (query, document) pairs into a relevance
+/// probability, in process, with no network call and no tokens.
+///
+/// An enum, not a `dyn` trait, for the same reason [`crate::Provider`] is:
+/// the set is closed and small, and the async arm would otherwise cost a
+/// boxed future per call.
+pub enum LocalJudge {
+ /// The deterministic lexical scorer ([`lexical_scores`]). Always
+ /// available, microseconds per page, no semantic signal — see the
+ /// module header for the honest scope.
+ Lexical,
+ /// The node's local decision head as a cross-encoder: an NLI pair score
+ /// of the document against "relevant to the query" and its negation.
+ /// Feature `judge-local`.
+ #[cfg(feature = "judge-local")]
+ Model(ModelJudge),
+}
+
+impl LocalJudge {
+ /// What the response's `judged.scorer` says. A caller must be able to
+ /// see WHICH judge judged, because the two arms are different scorers
+ /// with different ceilings.
+ pub fn name(&self) -> &'static str {
+ match self {
+ Self::Lexical => "lexical",
+ #[cfg(feature = "judge-local")]
+ Self::Model(_) => "model",
+ }
+ }
+
+ /// The model id the response's `judged.model` reports — on the model arm
+ /// only. The lexical arm is arithmetic, not a model, and must never have a
+ /// model name next to it: a response naming a "model" that is a formula
+ /// is the silent-fake this crate refuses everywhere.
+ pub fn model_id(&self) -> Option<&'static str> {
+ match self {
+ Self::Lexical => None,
+ #[cfg(feature = "judge-local")]
+ Self::Model(_) => Some(ModelJudge::MODEL_ID),
+ }
+ }
+
+ /// Score every candidate into a relevance probability. One [`Scored`]
+ /// per candidate, input order, ordinals untouched. Empty input is empty
+ /// output without running anything.
+ pub async fn judge(
+ &self,
+ query: &str,
+ candidates: &[Candidate],
+ ) -> Result<Vec<Scored>, RerankError> {
+ if candidates.is_empty() {
+ return Ok(Vec::new());
+ }
+ match self {
+ Self::Lexical => Ok(lexical_scores(query, candidates)),
+ #[cfg(feature = "judge-local")]
+ Self::Model(m) => m.judge(query, candidates).await,
+ }
+ }
+}
+
+/// The semantic arm: [`xerj_ai::decide::DecideHandle`] scoring one
+/// (document, hypothesis) pair per hit.
+///
+/// The handle shares the process-wide model registry with the decide tier
+/// (`xerj-ai`'s `shared_decide_cell`), so a node that armed
+/// `XERJ_DECIDE_MODE=local` judges with the SAME loaded copy the decide
+/// ladder uses — one model in memory, two questions asked of it.
+#[cfg(feature = "judge-local")]
+pub struct ModelJudge {
+ handle: xerj_ai::decide::DecideHandle,
+}
+
+#[cfg(feature = "judge-local")]
+impl ModelJudge {
+ /// The model id the response reports. It is the decide head's own id —
+ /// this arm IS that head asked a second question — and never a hosted
+ /// provider's name.
+ pub const MODEL_ID: &'static str = "xerj-decide-local-1";
+
+ /// Arm the judge on a model directory holding `config.json`,
+ /// `tokenizer.json` and `model.safetensors`. Loading is lazy (first
+ /// judged search) and shared process-wide with the decide tier.
+ pub fn from_model_dir(model_dir: PathBuf) -> Self {
+ Self {
+ handle: xerj_ai::decide::DecideHandle::new(xerj_ai::decide::DecideConfig { model_dir }),
+ }
+ }
+
+ /// The configured model directory, for diagnostics and error bodies.
+ pub fn model_dir(&self) -> &std::path::Path {
+ self.handle.model_dir()
+ }
+
+ /// Score every candidate: premise = the document (title then text, the
+ /// same concatenation the hosted provider's wire format sends),
+ /// hypotheses = the decide head's own relevance statement and its
+ /// negation, built with the head's trained templates
+ /// ([`xerj_ai::decide::hypothesis`] / `negation_hypothesis`) — the pair
+ /// shape that head was trained against, so a checkpoint trained for
+ /// decide scores correctly here without a second template to maintain.
+ ///
+ /// `p_relevant` is the entailment probability of the positive
+ /// hypothesis renormalised against its negation — the head's
+ /// `score_blocking` already returns each request's hypotheses summed to
+ /// 1, so `probs[0]` is that comparison's answer.
+ pub async fn judge(
+ &self,
+ query: &str,
+ candidates: &[Candidate],
+ ) -> Result<Vec<Scored>, RerankError> {
+ if candidates.is_empty() {
+ return Ok(Vec::new());
+ }
+ let label = format!("a passage relevant to the question: {query}");
+ let requests: Vec<xerj_ai::decide::ScoreRequest> = candidates
+ .iter()
+ .map(|c| {
+ let mut premise = String::new();
+ if let Some(t) = &c.title {
+ premise.push_str(t);
+ premise.push_str(". ");
+ }
+ premise.push_str(&c.text);
+ xerj_ai::decide::ScoreRequest {
+ premise,
+ hypotheses: vec![
+ xerj_ai::decide::hypothesis(&label),
+ xerj_ai::decide::negation_hypothesis(&label),
+ ],
+ }
+ })
+ .collect();
+ let scored = self.handle.score(requests).await.map_err(|e| {
+ RerankError::LocalModel(format!(
+ "the judge model at {} could not score this page: {e:#}",
+ self.model_dir().display()
+ ))
+ })?;
+ // One probability per request in order; the interface guarantees the
+ // lengths, and a request that somehow came back short is a contract
+ // break, not a zero score to invent.
+ if scored.len() != candidates.len() {
+ return Err(RerankError::LocalModel(format!(
+ "the judge model returned {} scores for {} documents",
+ scored.len(),
+ candidates.len()
+ )));
+ }
+ Ok(candidates
+ .iter()
+ .zip(scored)
+ .filter_map(|(c, probs)| {
+ probs.first().map(|&p| Scored {
+ ordinal: c.ordinal,
+ score: (p as f64).clamp(0.0, 1.0),
+ })
+ })
+ .collect())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn cand(ordinal: usize, title: Option<&str>, text: &str) -> Candidate {
+ Candidate {
+ ordinal,
+ title: title.map(str::to_string),
+ text: text.to_string(),
+ }
+ }
+
+ // ── the block parser ────────────────────────────────────────────────────
+
+ #[test]
+ fn parses_the_documented_block() {
+ let cfg = JudgeConfig::from_json(&json!({"local": true, "min_p": 0.5})).unwrap();
+ assert_eq!(cfg.min_p, Some(0.5));
+ assert_eq!(cfg.query, None);
+
+ // `local` may be omitted (there is only one judge), `{}` means
+ // "judge with defaults", and the ends of the range are valid —
+ // the same acceptance shape `rerank` gives `{}` and `min_score`.
+ let cfg = JudgeConfig::from_json(&json!({})).unwrap();
+ assert_eq!(cfg.min_p, None);
+ assert!(JudgeConfig::from_json(&json!({"min_p": 0.0})).is_ok());
+ assert!(JudgeConfig::from_json(&json!({"min_p": 1.0})).is_ok());
+
+ let cfg = JudgeConfig::from_json(&json!({"query": "why did the vpn drop?"})).unwrap();
+ assert_eq!(cfg.query.as_deref(), Some("why did the vpn drop?"));
+ }
+
+ #[test]
+ fn unknown_field_is_rejected_not_ignored() {
+ let err = JudgeConfig::from_json(&json!({"min_score": 0.5})).unwrap_err();
+ assert!(matches!(err, RerankError::Config(_)));
+ assert!(
+ err.to_string().contains("min_score"),
+ "the misspelling must be named: {err}"
+ );
+ assert!(err.to_string().contains("min_p"), "{err}");
+ }
+
+ #[test]
+ fn local_false_is_a_refusal_not_a_silent_no_op() {
+ // A block that says `local: false` has no other judge to be: honouring
+ // it silently would be the accepted-and-ignored class, so the caller
+ // is told to remove the block instead.
+ let err = JudgeConfig::from_json(&json!({"local": false})).unwrap_err();
+ assert!(err.to_string().contains("cannot be false"), "{err}");
+ let err = JudgeConfig::from_json(&json!({"local": "yes"})).unwrap_err();
+ assert!(err.to_string().contains("must be true"), "{err}");
+ }
+
+ #[test]
+ fn min_p_must_be_a_probability() {
+ assert!(JudgeConfig::from_json(&json!({"min_p": 1.2})).is_err());
+ assert!(JudgeConfig::from_json(&json!({"min_p": -0.1})).is_err());
+ assert!(JudgeConfig::from_json(&json!({"min_p": "high"})).is_err());
+ }
+
+ #[test]
+ fn the_question_has_a_ceiling_and_cannot_be_blank() {
+ let err =
+ JudgeConfig::from_json(&json!({"query": "q".repeat(MAX_QUERY_CHARS + 1)})).unwrap_err();
+ assert!(err.to_string().contains("judge.query"), "{err}");
+ assert!(err.to_string().len() < 400, "never echo the value: {err}");
+ assert!(JudgeConfig::from_json(&json!({"query": "é".repeat(MAX_QUERY_CHARS)})).is_ok());
+ assert!(JudgeConfig::from_json(&json!({"query": " "})).is_err());
+ }
+
+ #[test]
+ fn a_megabyte_key_name_is_not_echoed_back() {
+ let mut block = serde_json::Map::new();
+ block.insert("k".repeat(1_000_000), json!(1));
+ let err = JudgeConfig::from_json(&Value::Object(block))
+ .unwrap_err()
+ .to_string();
+ assert!(
+ err.contains("unknown `judge` field"),
+ "{}",
+ &err[..200.min(err.len())]
+ );
+ assert!(err.len() < 1_000, "echoed {} bytes", err.len());
+ }
+
+ // ── the lexical scorer ──────────────────────────────────────────────────
+
+ #[test]
+ fn no_query_words_scores_zero_everywhere() {
+ let out = lexical_scores(
+ "",
+ &[cand(0, Some("t"), "body"), cand(1, None, "other body")],
+ );
+ assert_eq!(out.len(), 2);
+ assert!(out.iter().all(|s| s.score == 0.0));
+ // Punctuation-only queries have no words either.
+ let out = lexical_scores("?! — …", &[cand(3, None, "body")]);
+ assert_eq!(out[0].score, 0.0);
+ }
+
+ #[test]
+ fn scores_are_ordered_by_term_coverage_and_bounded() {
+ let q = "vitamin d supplementation bone density";
+ let cands = vec![
+ cand(0, Some("Cooking"), "vitamin rich vegetables for dinner"),
+ cand(
+ 1,
+ Some("Trial results"),
+ "vitamin d supplementation improved bone density in the treatment group",
+ ),
+ cand(2, None, "nothing about the topic at all"),
+ ];
+ let out = lexical_scores(q, &cands);
+ assert_eq!(
+ out.iter().map(|s| s.ordinal).collect::<Vec<_>>(),
+ vec![0, 1, 2],
+ "input order preserved; the STAGE sorts"
+ );
+ let by = |o: usize| out.iter().find(|s| s.ordinal == o).unwrap().score;
+ assert_eq!(by(2), 0.0, "no query term present scores exactly 0");
+ assert!(by(1) > by(0), "full coverage beats one word: {:?}", out);
+ for s in &out {
+ assert!((0.0..=1.0).contains(&s.score), "{s:?}");
+ }
+ }
+
+ #[test]
+ fn a_rare_term_outweighs_a_term_every_document_shares() {
+ // Two docs, two query terms: "shared" is in both (df = 2), "unique"
+ // in one (df = 1). The doc holding the rare term must score higher
+ // than an equally-long doc holding only the common one — this is
+ // the window-local IDF doing its work.
+ let q = "shared unique";
+ let cands = vec![
+ cand(0, None, "shared shared shared"),
+ cand(1, None, "unique shared"),
+ ];
+ let out = lexical_scores(q, &cands);
+ assert!(
+ out[1].score > out[0].score,
+ "rare term must outweigh common: {out:?}"
+ );
+ }
+
+ #[test]
+ fn deterministic_and_repeatable() {
+ let q = "bone density trial results vitamin";
+ let cands: Vec<Candidate> = (0..30)
+ .map(|i| {
+ cand(
+ i,
+ Some(&format!("title {i}")),
+ &format!(
+ "body {i} with varying vitamin d bone density content {}",
+ "x".repeat(i)
+ ),
+ )
+ })
+ .collect();
+ let a = lexical_scores(q, &cands);
+ let b = lexical_scores(q, &cands);
+ assert_eq!(a, b, "same inputs must judge to the same probabilities");
+ assert!(a.iter().all(|s| (0.0..=1.0).contains(&s.score)));
+ }
+
+ /// The scorer's own cost for the issue's stated page (top-30), measured
+ /// here so the PR can state it honestly: the ≤ 40 ms p50 figure in the
+ /// issue is a RELEASE-time gate over the whole stage, not this micro
+ /// figure — but this is the part this PR adds.
+ #[test]
+ fn a_top_30_page_scores_in_microseconds() {
+ let q = "vitamin d supplementation and bone density in the treatment group";
+ let cands: Vec<Candidate> = (0..30)
+ .map(|i| {
+ cand(
+ i,
+ Some(&format!("doc title {i}")),
+ &("vitamin d bone density body text. ".repeat(50)),
+ )
+ })
+ .collect();
+ let started = std::time::Instant::now();
+ let out = lexical_scores(q, &cands);
+ let elapsed = started.elapsed();
+ assert_eq!(out.len(), 30);
+ assert!(
+ elapsed < std::time::Duration::from_millis(40),
+ "scoring 30 documents took {elapsed:?}"
+ );
+ }
+
+ #[test]
+ fn the_lexical_arm_judges_without_erring_and_names_itself() {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .build()
+ .unwrap();
+ let out = rt
+ .block_on(LocalJudge::Lexical.judge("q", &[cand(0, None, "q text")]))
+ .unwrap();
+ assert_eq!(
+ out,
+ vec![Scored {
+ ordinal: 0,
+ score: out[0].score
+ }]
+ );
+ assert_eq!(LocalJudge::Lexical.name(), "lexical");
+ // Empty page: no work, no error.
+ let out = rt.block_on(LocalJudge::Lexical.judge("q", &[])).unwrap();
+ assert!(out.is_empty());
+ }
+
+ // ── the model arm (compiled only with `judge-local`) ────────────────────
+
+ /// Plumbing only, against the tiny untrained fixture: the arm scores,
+ /// returns one bounded probability per candidate in order, and reuses
+ /// the decide loader's shared registry. Says NOTHING about quality —
+ /// the fixture is reproducible arithmetic, not judgement (see
+ /// `xerj-ai`'s `decide::testing`).
+ #[cfg(feature = "judge-local")]
+ #[test]
+ fn the_model_arm_scores_a_page_through_the_decide_loader() {
+ let dir = tempfile::tempdir().expect("tempdir").keep();
+ xerj_ai::decide::testing::write_fixture(&dir).expect("write decide fixture");
+ let judge = ModelJudge::from_model_dir(dir.clone());
+ assert_eq!(judge.model_dir(), dir.as_path());
+ assert_eq!(ModelJudge::MODEL_ID, "xerj-decide-local-1");
+
+ let cands = vec![
+ cand(0, Some("Bone health"), "vitamin d and bone density"),
+ cand(1, None, "cooking vegetables"),
+ ];
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let out = rt
+ .block_on(judge.judge("vitamin d bone density", &cands))
+ .unwrap();
+ assert_eq!(out.len(), 2);
+ assert_eq!(out[0].ordinal, 0);
+ assert_eq!(out[1].ordinal, 1);
+ assert!(
+ out.iter().all(|s| (0.0..=1.0).contains(&s.score)),
+ "{out:?}"
+ );
+ // Deterministic: the same page judges to the same probabilities.
+ let again = rt
+ .block_on(judge.judge("vitamin d bone density", &cands))
+ .unwrap();
+ assert_eq!(out, again);
+ assert_eq!(LocalJudge::Model(judge).name(), "model");
+ }
+
+ /// A missing model directory is a surfaced 503-class fault, never a
+ /// silent fall back to the lexical scorer.
+ #[cfg(feature = "judge-local")]
+ #[test]
+ fn a_missing_model_dir_surfaces_as_local_model() {
+ let judge = ModelJudge::from_model_dir(std::path::PathBuf::from("/nonexistent/judge"));
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let err = rt
+ .block_on(judge.judge("q", &[cand(0, None, "text")]))
+ .unwrap_err();
+ assert!(matches!(err, RerankError::LocalModel(_)), "{err}");
+ assert!(err.to_string().contains("config.json"), "{err}");
+ assert_eq!(err.policy(), crate::Policy::Surface);
+ }
+}
diff --git a/engine/crates/xerj-rerank/src/lib.rs b/engine/crates/xerj-rerank/src/lib.rs
index 4074159ec..d7451b50d 100644
--- a/engine/crates/xerj-rerank/src/lib.rs
+++ b/engine/crates/xerj-rerank/src/lib.rs
@@ -65,6 +65,8 @@ use std::time::Duration;
use serde::Deserialize;
use serde_json::{json, Map, Value};
+pub mod judge;
+
/// Hard ceiling on documents per Jev request.
///
/// The `hev/jev-rerank` README reports a ~32k-token request budget and
@@ -197,6 +199,12 @@ pub enum RerankError {
Config(String),
#[error("rerank deadline exceeded after {elapsed_ms}ms")]
Deadline { elapsed_ms: u128 },
+ /// The local judge's model could not be loaded or could not score the
+ /// page ([`crate::judge::ModelJudge`]). Operator state, not the caller's
+ /// request — and never a reason to silently hand back unjudged hits or
+ /// substitute the lexical scorer the response did not name.
+ #[error("the local judge model is unavailable: {0}")]
+ LocalModel(String),
/// The operator switched reranking off (`[rerank] enabled = false`).
/// Distinct from [`Self::MissingKey`]: that one is "not set up yet", this
/// one is "deliberately never" — reranking sends document text to a third
@@ -230,6 +238,7 @@ impl RerankError {
| Self::UnknownProvider(_)
| Self::Config(_)
| Self::Malformed(_)
+ | Self::LocalModel(_)
| Self::DisabledByOperator
| Self::Status { .. } => Policy::Surface,
}
@@ -575,7 +584,7 @@ pub fn longer_than(s: &str, max: usize) -> bool {
}
/// How much of a caller-supplied NAME (an unknown key) a refusal echoes back.
-const MAX_ECHOED_NAME_CHARS: usize = 64;
+pub(crate) const MAX_ECHOED_NAME_CHARS: usize = 64;
/// A `rerank.<key>` string under its server-side ceiling.
///
diff --git a/engine/crates/xerj-server/src/grpc.rs b/engine/crates/xerj-server/src/grpc.rs
index c4e5bf015..cec0e55a9 100644
--- a/engine/crates/xerj-server/src/grpc.rs
+++ b/engine/crates/xerj-server/src/grpc.rs
@@ -134,11 +134,17 @@ impl XerjSearch for GrpcService {
// Refused by name instead. Checked before the bare-clause wrap below,
// which would otherwise bury the key inside `query`.
// `carries_rerank`, the rule every HTTP surface uses: `"rerank": null`
- // is an absent block, not a rerank request.
- if xerj_api::rerank_stage::carries_rerank(&body) {
- return Err(Status::invalid_argument(
- xerj_api::rerank_stage::unsupported_reason("the gRPC Search RPC"),
- ));
+ // is an absent block, not a rerank request. A `judge` block (issue
+ // #1060) is refused for the same reason, through the same
+ // second-stage check the raw-JSON HTTP surfaces use — naming the
+ // block the body actually carries.
+ if xerj_api::judge_stage::carries_second_stage(&body) {
+ let reason = if xerj_api::rerank_stage::carries_rerank(&body) {
+ xerj_api::rerank_stage::unsupported_reason("the gRPC Search RPC")
+ } else {
+ xerj_api::judge_stage::unsupported_reason("the gRPC Search RPC")
+ };
+ return Err(Status::invalid_argument(reason));
}
if body.get("query").is_none() {
body = serde_json::json!({ "query": body });
diff --git a/landing/docs/agents/schemas/mcp-tools.json b/landing/docs/agents/schemas/mcp-tools.json
index 5d1a63a58..c28a9cdd3 100644
--- a/landing/docs/agents/schemas/mcp-tools.json
+++ b/landing/docs/agents/schemas/mcp-tools.json
@@ -2,7 +2,7 @@
"tools": [
{
"name": "xerj_search",
- "description": "Full-text / keyword / structured search over a XERJ index. Proxies POST /{index}/_search. Pass `query` as a PLAIN STRING for the definition-first code search (ranks the file that defines a symbol above files that merely mention it — the right default for code), or as an ES query-DSL object (e.g. {\"term\":{\"status\":\"open\"}}, or a bool clause) for structured queries. Omit `query` for match_all. Optional `rerank` adds a second stage that re-judges the top hits with an external relevance model — read that argument's description before using it: it needs a provider the operator configured, and it sends document text off the machine. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
+ "description": "Full-text / keyword / structured search over a XERJ index. Proxies POST /{index}/_search. Pass `query` as a PLAIN STRING for the definition-first code search (ranks the file that defines a symbol above files that merely mention it — the right default for code), or as an ES query-DSL object (e.g. {\"term\":{\"status\":\"open\"}}, or a bool clause) for structured queries. Omit `query` for match_all. Optional `rerank` adds a second stage that re-judges the top hits with an external relevance model — read that argument's description before using it: it needs a provider the operator configured, and it sends document text off the machine. Optional `judge` re-judges the returned hits LOCALLY (zero tokens, nothing leaves the node) and adds a `_p_relevant` probability per hit — see that argument's description for its honest ceiling. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -38,6 +38,13 @@
],
"description": "Optional second-stage reranking. XERJ sends the text of the top `window` hits plus the question to an external relevance model, which returns a calibrated probability (0..1) per document. Hits are reordered by it, that probability REPLACES `_score`, and `min_score` prunes by it — an absolute relevance cut, which a BM25 score cannot give you. WORTH IT WHEN: a natural-language question over prose where the right document is probably in the top 30 but not at the top; or you need a yes/no relevance threshold rather than a relative rank. NOT WORTH IT FOR: exact identifier or symbol lookups (the first stage already ranks the definition first), filters, sorted or aggregation-only requests — it adds a paid network round-trip per search. NEEDS A CONFIGURED PROVIDER: the node's operator must have set a provider key (`[rerank] api_key` or TYPESAFE_API_KEY). You cannot supply one in this call. Without it the search fails with HTTP 503 `rerank_exception`; do not retry, repeat the search without `rerank`. HTTP 403 means the operator disabled it. PRIVACY: this is the only search-time feature that sends document text off the node. Proxy embeddings (`[embedding] default_endpoint`) and the WAL tap also send text off the node when an operator configures them, so never tell a user that nothing else leaves the machine — do not use rerank on data that must stay local unless the user has agreed. Only fields the response returns are sent, so `_source` filtering also limits what leaves. Pass true or {} for defaults, or an object with: `query` (string): the natural-language question to judge documents against. Filled in for you when the tool's `query` is a plain string (that mode also returns the matching passage, which leads the text the judge reads); REQUIRED when the tool's `query` is a `bool`, a `knn`, or anything else with no single question in it (the node refuses rather than guess). `window` (int, default 30, max 300): how many top hits are judged — every one is a paid judgement; `from`+`size` must fit inside it. `min_score` (0..1): drop hits judged below this probability. `fields` (string[]): which returned fields to send; default is every returned string field. `instructions` (string): override what \"relevant\" means for this corpus. `timeout_ms` (int, default 10000). `model` (string). Cannot be combined with `sort`. READ THE RESPONSE'S `_rerank` BLOCK: `applied: true` means the order is the judge's and every `_score` is a probability or null (a hit with no verdict, counted in `unjudged`, sorts last); `applied: false` means the provider missed the deadline or answered nothing usable, and you are looking at the engine's own order with engine scores, stated in `reason`."
},
+ "judge": {
+ "type": [
+ "object",
+ "boolean"
+ ],
+ "description": "Optional LOCAL judging of the returned hits — the zero-token second stage. The node re-scores every hit it was about to return, in process, against the question: no provider, no API key, no network call, and NO document text leaves the machine (unlike `rerank`). Each judged hit carries `_p_relevant`, a 0..1 relevance probability on an absolute scale — the engine's `_score` stays untouched beside it — and hits are reordered by it. `min_p` (0..1) drops hits judged below the probability and counts them in the response's `judged` block (`{kept, dropped}`): an absolute relevance cut you can threshold, which a BM25 score cannot give you. USE IT WHEN: you want a per-hit relevance number or a keep/drop cut on every search without cost, or the data must stay on the node. HONEST CEILING: with no local model armed the judge is a deterministic lexical scorer — it re-reads the same words the first stage read, with page-local statistics, and has NO semantic signal. The response's `judged.scorer` says which judge judged (`lexical` or `model`); read it before treating `_p_relevant` as understanding. Prefer `rerank` (paid, egress) when semantic judgement is worth the cost. Pass true or {} for defaults, or an object with: `query` (string): the natural-language question to judge documents against. Filled in for you when the tool's `query` is a plain string; REQUIRED when the tool's `query` is a `bool`, a `knn`, or anything else with no single question in it (the node refuses rather than guess). `min_p` (0..1): drop hits judged below this probability. Cannot be combined with `sort`. READ THE RESPONSE'S `judged` BLOCK: `scorer` names the judge, `kept`/`dropped` count the page, `unjudged` counts hits with no text to judge (they carry no `_p_relevant` and sort last)."
+ },
"max_tokens": {
"type": "integer",
"minimum": 32,
@@ -131,7 +138,7 @@
},
{
"name": "xerj_hybrid_search",
- "description": "Hybrid search: fuse several sub-queries (e.g. a lexical `match` plus a vector `knn`) into one ranked list. Fusion is `rrf` (reciprocal-rank) or `linear` (weighted). Proxies POST /{index}/_search with {\"query\":{\"hybrid\":{\"queries\":[...],\"fusion\":...}}}. Optional `rerank` re-judges the fused top hits with an external relevance model; here `rerank.query` is REQUIRED, because a hybrid search has several sub-queries and no single question to read. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
+ "description": "Hybrid search: fuse several sub-queries (e.g. a lexical `match` plus a vector `knn`) into one ranked list. Fusion is `rrf` (reciprocal-rank) or `linear` (weighted). Proxies POST /{index}/_search with {\"query\":{\"hybrid\":{\"queries\":[...],\"fusion\":...}}}. Optional `rerank` re-judges the fused top hits with an external relevance model; here `rerank.query` is REQUIRED, because a hybrid search has several sub-queries and no single question to read. Optional `judge` re-judges the fused hits LOCALLY (zero tokens, nothing leaves the node); `judge.query` is REQUIRED for the same reason. Pass `max_tokens` to cap the response at a token budget (never exceeded; see that argument for the tokenizer and the locator-only rule).",
"inputSchema": {
"type": "object",
"properties": {
@@ -175,6 +182,13 @@
],
"description": "Optional second-stage reranking. XERJ sends the text of the top `window` hits plus the question to an external relevance model, which returns a calibrated probability (0..1) per document. Hits are reordered by it, that probability REPLACES `_score`, and `min_score` prunes by it — an absolute relevance cut, which a BM25 score cannot give you. WORTH IT WHEN: a natural-language question over prose where the right document is probably in the top 30 but not at the top; or you need a yes/no relevance threshold rather than a relative rank. NOT WORTH IT FOR: exact identifier or symbol lookups (the first stage already ranks the definition first), filters, sorted or aggregation-only requests — it adds a paid network round-trip per search. NEEDS A CONFIGURED PROVIDER: the node's operator must have set a provider key (`[rerank] api_key` or TYPESAFE_API_KEY). You cannot supply one in this call. Without it the search fails with HTTP 503 `rerank_exception`; do not retry, repeat the search without `rerank`. HTTP 403 means the operator disabled it. PRIVACY: this is the only search-time feature that sends document text off the node. Proxy embeddings (`[embedding] default_endpoint`) and the WAL tap also send text off the node when an operator configures them, so never tell a user that nothing else leaves the machine — do not use rerank on data that must stay local unless the user has agreed. Only fields the response returns are sent, so `_source` filtering also limits what leaves. Pass true or {} for defaults, or an object with: `query` (string, REQUIRED for this tool): the natural-language question to judge documents against. `window` (int, default 30, max 300): how many top hits are judged — every one is a paid judgement; `from`+`size` must fit inside it. `min_score` (0..1): drop hits judged below this probability. `fields` (string[]): which returned fields to send; default is every returned string field. `instructions` (string): override what \"relevant\" means for this corpus. `timeout_ms` (int, default 10000). `model` (string). Cannot be combined with `sort`. READ THE RESPONSE'S `_rerank` BLOCK: `applied: true` means the order is the judge's and every `_score` is a probability or null (a hit with no verdict, counted in `unjudged`, sorts last); `applied: false` means the provider missed the deadline or answered nothing usable, and you are looking at the engine's own order with engine scores, stated in `reason`."
},
+ "judge": {
+ "type": [
+ "object",
+ "boolean"
+ ],
+ "description": "Optional LOCAL judging of the returned hits — the zero-token second stage. The node re-scores every hit it was about to return, in process, against the question: no provider, no API key, no network call, and NO document text leaves the machine (unlike `rerank`). Each judged hit carries `_p_relevant`, a 0..1 relevance probability on an absolute scale — the engine's `_score` stays untouched beside it — and hits are reordered by it. `min_p` (0..1) drops hits judged below the probability and counts them in the response's `judged` block (`{kept, dropped}`): an absolute relevance cut you can threshold, which a BM25 score cannot give you. USE IT WHEN: you want a per-hit relevance number or a keep/drop cut on every search without cost, or the data must stay on the node. HONEST CEILING: with no local model armed the judge is a deterministic lexical scorer — it re-reads the same words the first stage read, with page-local statistics, and has NO semantic signal. The response's `judged.scorer` says which judge judged (`lexical` or `model`); read it before treating `_p_relevant` as understanding. Prefer `rerank` (paid, egress) when semantic judgement is worth the cost. Pass true or {} for defaults, or an object with: `query` (string, REQUIRED for this tool): the natural-language question to judge documents against. `min_p` (0..1): drop hits judged below this probability. Cannot be combined with `sort`. READ THE RESPONSE'S `judged` BLOCK: `scorer` names the judge, `kept`/`dropped` count the page, `unjudged` counts hits with no text to judge (they carry no `_p_relevant` and sort last)."
+ },
"max_tokens": {
"type": "integer",
"minimum": 32,