Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! The abstention gate (`min_confidence`, #361/#394): the router's side of it with fake predictors
//! (validation order, batch-wide application, nothing loaded on a bad threshold), the exact error
//! texts, and the recorded Python results for `confidence_probe.json` and the model cases.
//!
//! The probe is model-free: it holds the output of Python's own `confidence.py` functions. The
//! model-backed cases skip, with a `skip:` line, until the golden files carry them or without
//! `LAYA_ONNX_ROOT`.
mod common;
use common::routing_golden;
use indexmap::IndexMap;
use laya::answers::{ActionInfo, Answer, ChoiceAnswer, LayaResult, NoulAnswer, ScoreAnswer, Usage};
use laya::batch_prediction::BatchOptions;
use laya::confidence::{GATE_STATES, MinConfidence, bucket_keys};
use laya::error::{LayaError, Result};
use laya::laya_options::LayaCheckpoint;
use laya::laya_router::{
BatchRequest, LayaBatchPredictor, LayaPredictor, LayaRouter, LayaRouterOptions,
};
use laya::predict_options::PredictOptions;
use laya::questions::{Question, QuestionSet};
use serde_json::Value;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
// ---------------------------------------------------------------------- model-free
type Builds = Arc<Mutex<Vec<LayaCheckpoint>>>;
type Seen = Arc<Mutex<Vec<BatchOptions>>>;
/// A predictor whose answers carry the confidence its state names: the state `"0.8"` gives a
/// choice answer with `answer_confidence` 0.8 and a noul answer with the same.
struct Fake {
seen: Seen,
}
fn answers_for(state: &Value) -> LayaResult {
let confidence: f64 = state.as_str().and_then(|s| s.parse().ok()).unwrap_or(0.5);
let mut probabilities = IndexMap::new();
probabilities.insert("a".to_string(), confidence);
probabilities.insert("b".to_string(), 1.0 - confidence);
let choice = Answer::Choice(ChoiceAnswer::new(
"a",
probabilities,
0.1,
confidence,
ActionInfo::default(),
));
let noul = Answer::Noul(NoulAnswer::new(
confidence,
0.1,
confidence,
ActionInfo::default(),
));
let mut by_id = HashMap::new();
by_id.insert("pick".to_string(), choice);
by_id.insert("flag".to_string(), noul);
LayaResult::new(
"fake",
vec!["pick".to_string(), "flag".to_string()],
by_id,
Usage::default(),
)
}
impl LayaPredictor for Fake {
fn predict(&self, state: Value, _questions: &QuestionSet) -> Result<LayaResult> {
Ok(answers_for(&state))
}
fn as_batch_predictor(&self) -> Option<&dyn LayaBatchPredictor> {
Some(self)
}
}
impl LayaBatchPredictor for Fake {
fn predict_batch(
&self,
states: &[Value],
_questions: &QuestionSet,
options: &BatchOptions,
) -> Result<Vec<LayaResult>> {
self.seen.lock().unwrap().push(options.clone());
Ok(states.iter().map(answers_for).collect())
}
}
fn fixture() -> (LayaRouter, Builds, Seen) {
let builds: Builds = Arc::default();
let seen: Seen = Arc::default();
let (factory_builds, factory_seen) = (Arc::clone(&builds), Arc::clone(&seen));
let router = LayaRouter::with_factory(LayaRouterOptions::default(), move |checkpoint, _| {
factory_builds.lock().unwrap().push(checkpoint);
Ok(Arc::new(Fake {
seen: Arc::clone(&factory_seen),
}) as Arc<dyn LayaPredictor>)
})
.expect("a fake router never fails to construct");
(router, builds, seen)
}
fn questions() -> QuestionSet {
QuestionSet::new()
.with(
"pick",
Question::choice_labels("Which?", ["a", "b"]).unwrap(),
)
.unwrap()
}
fn scalar(threshold: f64) -> PredictOptions {
PredictOptions::new().min_confidence(MinConfidence::scalar(threshold).unwrap())
}
#[test]
fn router_predict_gates_with_the_options_threshold() {
let (router, _, _) = fixture();
let result = router
.predict_with_options(
"0.8",
&questions(),
Some("en"),
None,
None,
None,
&scalar(0.9),
)
.unwrap();
assert_eq!(result["pick"].abstention(), Some("abstained"));
assert!(result["pick"].low_confidence());
assert_eq!(result["pick"].abstention_threshold(), Some(0.9));
let passed = router
.predict_with_options(
"0.8",
&questions(),
Some("en"),
None,
None,
None,
&scalar(0.5),
)
.unwrap();
assert_eq!(passed["pick"].abstention(), Some("passed"));
assert!(passed.routing().is_some(), "the routing decision is kept");
}
#[test]
fn router_predict_without_a_threshold_reports_no_gate() {
let (router, _, _) = fixture();
for result in [
router
.predict("0.8", &questions(), Some("en"), None, None)
.unwrap(),
router
.predict_with_options(
"0.8",
&questions(),
Some("en"),
None,
None,
None,
&PredictOptions::default(),
)
.unwrap(),
] {
for (_, answer) in result.iter() {
assert!(!answer.low_confidence());
assert_eq!(answer.abstention(), None);
assert_eq!(answer.abstention_threshold(), None);
}
}
}
#[test]
fn router_predict_refuses_a_bad_threshold_before_routing_or_loading() {
let (router, builds, _) = fixture();
let bad = PredictOptions::new().min_confidence(MinConfidence::Scalar(1.5));
let error = router
.predict_with_options("0.8", &questions(), Some("en"), None, None, None, &bad)
.unwrap_err();
assert!(matches!(error, LayaError::InvalidMinConfidence(_)));
assert_eq!(
error.to_string(),
"min_confidence must be a float in [0.0, 1.0], got 1.5"
);
assert!(builds.lock().unwrap().is_empty(), "nothing was loaded");
// Router.predict checks the state first (router.py:1336), then the threshold (:1338).
let error = router
.predict_with_options(
Value::Null,
&questions(),
Some("en"),
None,
None,
None,
&bad,
)
.unwrap_err();
assert!(matches!(error, LayaError::InvalidState(_)), "{error}");
}
#[test]
fn router_predict_batch_gates_every_result_with_one_threshold() {
let (router, _, seen) = fixture();
let requests = [
BatchRequest::new("0.95", questions()).model("en"),
BatchRequest::new("0.2", questions()).model("multi"),
BatchRequest::new("0.8", questions()).model("en"),
];
let options = BatchOptions::new().min_confidence(MinConfidence::scalar(0.9).unwrap());
let results = router.predict_batch(&requests, &options).unwrap();
let states: Vec<_> = results
.iter()
.map(|r| (r["pick"].abstention(), r["flag"].abstention()))
.collect();
assert_eq!(
states,
[
(Some("passed"), Some("passed")),
(Some("abstained"), Some("abstained")),
(Some("abstained"), Some("abstained")),
]
);
for result in &results {
assert_eq!(result["pick"].abstention_threshold(), Some(0.9));
}
// The threshold is the router's to apply; it is no part of the grouping or of what the
// predictors are asked.
assert!(
seen.lock()
.unwrap()
.iter()
.all(|o| o.min_confidence.is_none())
);
assert_eq!(seen.lock().unwrap().len(), 2, "one call per checkpoint");
}
#[test]
fn router_predict_batch_refuses_a_bad_threshold_before_routing_or_loading() {
let (router, builds, _) = fixture();
// Even the routing error is not reached: the request names a checkpoint that does not exist.
let requests = [BatchRequest::new("0.8", questions()).model("no-such-model")];
let mut map = IndexMap::new();
map.insert("choice:1".to_string(), 0.5);
let options = BatchOptions::new().min_confidence(MinConfidence::Map(map));
let error = router.predict_batch(&requests, &options).unwrap_err();
assert!(
matches!(error, LayaError::InvalidMinConfidence(_)),
"{error}"
);
assert!(builds.lock().unwrap().is_empty());
}
#[test]
fn router_predict_batch_with_no_requests_still_validates() {
let (router, _, _) = fixture();
let options = BatchOptions::new().min_confidence(MinConfidence::Scalar(-1.0));
assert!(router.predict_batch(&[], &options).is_err());
assert!(
router
.predict_batch(&[], &BatchOptions::new())
.unwrap()
.is_empty()
);
}
#[test]
fn the_public_vocabulary_is_python_s() {
assert_eq!(GATE_STATES, ["passed", "abstained", "unevaluated"]);
let keys = bucket_keys();
assert_eq!(keys.len(), 12);
for qtype in ["choice", "score", "noul"] {
for size in ["2", "3-5", "6-10", "11+"] {
assert!(keys.contains(&format!("{qtype}:{size}")));
}
}
}
// ---------------------------------------------------------------------- confidence_probe.json
/// Python's own `laya/confidence.py` over the shared synthetic inputs of
/// `tools/confidence_cases.py`. A float field is a JSON number, or "nan"/"inf"/"-inf".
fn probe() -> Option<Value> {
let probe = routing_golden::load("confidence_probe.json");
if probe.is_none() {
eprintln!("skip: routing golden not found (tests/golden/routing/confidence_probe.json)");
}
probe
}
fn try_float(value: &Value) -> Option<f64> {
match value {
Value::Number(n) => n.as_f64(),
Value::String(s) => match s.as_str() {
"nan" => Some(f64::NAN),
"inf" => Some(f64::INFINITY),
"-inf" => Some(f64::NEG_INFINITY),
_ => None,
},
_ => None,
}
}
fn float(value: &Value) -> f64 {
try_float(value).unwrap_or_else(|| panic!("not a float: {value}"))
}
/// A float field the answer may not have reported: a typed answer reads it as NaN.
fn reported(answer: &Value, key: &str) -> f64 {
answer.get(key).map_or(f64::NAN, float)
}
/// A typed answer for an entry of the probe, or `None` for the shapes a typed answer cannot hold
/// (`"typed": false`: a boolean or text "probability", a non-integer score level).
fn build_answer(entry: &Value) -> Option<Answer> {
let confidence = reported(entry, "confidence");
let answer_confidence = reported(entry, "answer_confidence");
let action = ActionInfo::default();
let probabilities = entry.get("probabilities").and_then(Value::as_object);
match entry["type"].as_str()? {
"choice" => {
let mut map = IndexMap::new();
for (label, value) in probabilities? {
map.insert(label.clone(), try_float(value)?);
}
let first = map.keys().next().cloned().unwrap_or_default();
Some(Answer::Choice(ChoiceAnswer::new(
first,
map,
confidence,
answer_confidence,
action,
)))
}
"score" => {
let mut levels: Vec<(i64, f64)> = Vec::new();
for (level, value) in probabilities? {
levels.push((level.parse().ok()?, try_float(value)?));
}
levels.sort_by_key(|(level, _)| *level);
Some(Answer::Score(ScoreAnswer::new(
0.0,
Vec::new(),
levels.into_iter().map(|(_, p)| p).collect(),
confidence,
answer_confidence,
action,
)))
}
"noul" => Some(Answer::Noul(NoulAnswer::new(
float(&entry["noul"]),
confidence,
answer_confidence,
action,
))),
_ => None,
}
}
fn result_of(answers: Vec<(String, Answer)>) -> LayaResult {
LayaResult::new(
"probe",
answers.iter().map(|(id, _)| id.clone()).collect(),
answers.into_iter().collect(),
Usage::default(),
)
}
/// A threshold as the probe spells it: null (unset), a float, or an ordered bucket map.
fn threshold(value: &Value) -> Option<MinConfidence> {
match value {
Value::Null => None,
Value::Object(map) => Some(
MinConfidence::map(map.iter().map(|(k, v)| (k.clone(), float(v)))).expect("valid map"),
),
other => Some(MinConfidence::scalar(float(other)).expect("valid scalar")),
}
}
/// A router whose predictor hands back whatever result is in the slot, so a test can feed a
/// gated result back in, as a reused payload is.
struct Slot(Mutex<LayaResult>);
impl LayaPredictor for Slot {
fn predict(&self, _state: Value, _questions: &QuestionSet) -> Result<LayaResult> {
Ok(self.0.lock().unwrap().clone())
}
}
fn slot_router(result: LayaResult) -> (LayaRouter, Arc<Slot>) {
let slot = Arc::new(Slot(Mutex::new(result)));
let shared = Arc::clone(&slot);
let router = LayaRouter::with_factory(LayaRouterOptions::default(), move |_, _| {
Ok(Arc::clone(&shared) as Arc<dyn LayaPredictor>)
})
.expect("a fake router never fails to construct");
(router, slot)
}
/// Gate the slot's result at `min_confidence` through `Router::predict_with_options`, and store the
/// gated result back, so the next call gates a reused one.
fn gate_slot(
router: &LayaRouter,
slot: &Slot,
min_confidence: Option<MinConfidence>,
) -> LayaResult {
let mut options = PredictOptions::new();
options.min_confidence = min_confidence;
let result = router
.predict_with_options(
"state",
&questions(),
Some("en"),
None,
None,
None,
&options,
)
.expect("the gate does not fail on a valid threshold");
*slot.0.lock().unwrap() = result.clone();
result
}
fn assert_same_float(got: f64, want: f64, context: &str) {
assert!(
got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()),
"{context}: got {got:?}, python gave {want:?}"
);
}
/// What the gate wrote on each answer must be exactly what Python wrote: `low_confidence` only
/// when flagged, `abstention` and `abstention_threshold` only when a threshold was set.
fn assert_verdicts(result: &LayaResult, expected: &Value, context: &str) {
let expected = expected.as_object().expect("expected verdicts");
assert_eq!(result.len(), expected.len(), "{context}: answer count");
for (id, want) in expected {
let answer = &result[id.as_str()];
let where_answer = format!("{context} / {id}");
assert_eq!(
want.get("low_confidence").is_some(),
answer.low_confidence(),
"{where_answer}: low_confidence"
);
if let Some(flag) = want.get("low_confidence") {
assert_eq!(flag, &Value::Bool(true), "{where_answer}: only ever true");
}
assert_eq!(
want.get("abstention").and_then(Value::as_str),
answer.abstention(),
"{where_answer}: abstention"
);
match (
want.get("abstention_threshold"),
answer.abstention_threshold(),
) {
(None, None) => {}
(Some(want), Some(got)) => {
assert_same_float(got, float(want), &format!("{where_answer}: threshold"));
}
(want, got) => panic!("{where_answer}: threshold {got:?} vs python {want:?}"),
}
}
}
fn gate_answers(probe: &Value) -> Vec<(String, Answer)> {
probe["gate_answers"]
.as_object()
.expect("gate_answers")
.iter()
.map(|(id, entry)| {
(
id.clone(),
build_answer(entry).expect("every gate answer is typed"),
)
})
.collect()
}
#[test]
fn probe_vocabulary_matches_python() {
let Some(probe) = probe() else { return };
let mut keys = bucket_keys();
keys.sort();
let expected: Vec<&str> = probe["bucket_keys"]
.as_array()
.unwrap()
.iter()
.map(|k| k.as_str().unwrap())
.collect();
assert_eq!(keys, expected);
let states: Vec<&str> = probe["gate_states"]
.as_array()
.unwrap()
.iter()
.map(|k| k.as_str().unwrap())
.collect();
assert_eq!(GATE_STATES.to_vec(), states);
}
#[test]
fn probe_scalar_thresholds_validate_as_python_does() {
let Some(probe) = probe() else { return };
let cases = probe["scalars"].as_array().unwrap();
assert!(!cases.is_empty());
for case in cases {
let value = float(&case["value"]);
let got = MinConfidence::scalar(value);
if case["valid"].as_bool().unwrap() {
let want = float(&case["result"]);
match got {
Ok(MinConfidence::Scalar(kept)) => assert_same_float(kept, want, "scalar"),
other => panic!("{value:?}: python accepted it, got {other:?}"),
}
} else {
let error = got.expect_err(&format!("{value:?}: python rejected it"));
assert!(matches!(error, LayaError::InvalidMinConfidence(_)));
assert_eq!(
error.to_string(),
case["error_message"].as_str().unwrap(),
"{value:?}"
);
}
}
}
#[test]
fn probe_threshold_maps_validate_as_python_does() {
let Some(probe) = probe() else { return };
let cases = probe["maps"].as_array().unwrap();
assert!(!cases.is_empty());
for case in cases {
let pairs: Vec<(String, f64)> = case["pairs"]
.as_array()
.unwrap()
.iter()
.map(|pair| (pair[0].as_str().unwrap().to_string(), float(&pair[1])))
.collect();
let got = MinConfidence::map(pairs.clone());
if case["valid"].as_bool().unwrap() {
let Ok(MinConfidence::Map(kept)) = got else {
panic!("{pairs:?}: python accepted it, got {got:?}");
};
// A repeated key keeps its first position and its last value, as a dict does.
let want: Vec<(String, f64)> = case["result"]
.as_array()
.unwrap()
.iter()
.map(|pair| (pair[0].as_str().unwrap().to_string(), float(&pair[1])))
.collect();
let got: Vec<(String, f64)> = kept.into_iter().collect();
assert_eq!(got.len(), want.len(), "{pairs:?}");
for ((got_key, got_value), (want_key, want_value)) in got.iter().zip(&want) {
assert_eq!(got_key, want_key, "{pairs:?}");
assert_same_float(*got_value, *want_value, &format!("{pairs:?}"));
}
} else {
let error = got.expect_err(&format!("{pairs:?}: python rejected it"));
assert!(matches!(error, LayaError::InvalidMinConfidence(_)));
assert_eq!(
error.to_string(),
case["error_message"].as_str().unwrap(),
"{pairs:?}"
);
}
}
}
#[test]
fn probe_option_buckets_match_python() {
let Some(probe) = probe() else { return };
let cases = probe["buckets"].as_array().unwrap();
assert!(!cases.is_empty());
for case in cases {
let answer = build_answer(&case["answer"]).expect("a typed answer");
assert_eq!(
answer.option_bucket().as_deref(),
case["bucket"].as_str(),
"{}",
case["answer"]
);
}
}
#[test]
fn probe_resolved_thresholds_match_python() {
let Some(probe) = probe() else { return };
let cases = probe["resolve"].as_array().unwrap();
assert!(!cases.is_empty());
for case in cases {
let answer = build_answer(&case["answer"]).expect("a typed answer");
let (router, slot) = slot_router(result_of(vec![("q".to_string(), answer)]));
let gated = gate_slot(&router, &slot, threshold(&case["thresholds"]));
assert_same_float(
gated["q"].abstention_threshold().expect("a gate ran"),
float(&case["result"]),
&format!("{} under {}", case["answer"], case["thresholds"]),
);
}
}
#[test]
fn probe_gate_verdicts_match_python() {
let Some(probe) = probe() else { return };
let cases = probe["gate"].as_array().unwrap();
assert!(!cases.is_empty());
for case in cases {
let (router, slot) = slot_router(result_of(gate_answers(&probe)));
let gated = gate_slot(&router, &slot, threshold(&case["threshold"]));
assert_verdicts(
&gated,
&case["expected"],
&format!("threshold {}", case["threshold"]),
);
}
}
/// #910: python's reused payload carried a stale flag across thresholds. A result here is a
/// value, but one gated and gated again must still end up as the last threshold alone decides.
#[test]
fn probe_reused_results_never_keep_an_earlier_verdict() {
let Some(probe) = probe() else { return };
let sequences = probe["reuse"].as_array().unwrap();
assert!(!sequences.is_empty());
for (index, sequence) in sequences.iter().enumerate() {
let (router, slot) = slot_router(result_of(gate_answers(&probe)));
for (step, case) in sequence["steps"].as_array().unwrap().iter().enumerate() {
let gated = gate_slot(&router, &slot, threshold(&case["threshold"]));
assert_verdicts(
&gated,
&case["expected"],
&format!("reuse {index} step {step} at {}", case["threshold"]),
);
}
}
}
#[test]
fn probe_jev_confidence_matches_python() {
let Some(probe) = probe() else { return };
let cases = probe["jev"].as_array().unwrap();
assert!(!cases.is_empty());
let (mut typed, mut untyped) = (0, 0);
for case in cases {
let name = case["name"].as_str().unwrap();
let want = match &case["result"] {
Value::Null => None,
value => Some(float(value)),
};
// The shapes a typed answer cannot hold are the ones python refuses: no result.
let Some(answer) = build_answer(&case["answer"]) else {
assert!(
!case["typed"].as_bool().unwrap(),
"{name}: should be buildable"
);
assert_eq!(
want, None,
"{name}: python refuses what a typed answer cannot hold"
);
untyped += 1;
continue;
};
assert!(
case["typed"].as_bool().unwrap(),
"{name}: typed answer built from an untyped case"
);
typed += 1;
match (answer.jev_confidence(), want) {
(None, None) => {}
(Some(got), Some(want)) => assert_same_float(got, want, name),
(got, want) => panic!("{name}: got {got:?}, python gave {want:?}"),
}
}
assert!(
typed > 60 && untyped >= 4,
"typed {typed}, untyped {untyped}"
);
}
// ---------------------------------------------------------------------- model-backed
/// The `min_confidence` cases of the checkpoint goldens (`min_confidence_*` for `system_one`,
/// `batch_min_confidence_*` for `predict_batch`) against the real model. The parity tests
/// (`predict_parity_tests`, `predict_batch_tests`) run every recorded case with its options; this
/// is what says the cases exist at all, since goldens recorded before the gate hold none and
/// would otherwise pass those tests without ever exercising it.
#[test]
fn the_recorded_min_confidence_cases_reproduce() {
let mut checked_any = false;
for checkpoint in common::engines::checkpoints() {
let Some(engine) = common::engines::load(checkpoint) else {
continue;
};
let golden = common::golden_data::CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
eprintln!("skip: no golden data for '{}'", checkpoint.subdir());
continue;
}
let single: Vec<_> = golden
.success_cases()
.into_iter()
.filter(|case| case.name.starts_with("min_confidence_"))
.collect();
let batch: Vec<_> = golden
.batch_cases()
.into_iter()
.filter(|case| case.name.starts_with("batch_min_confidence_"))
.collect();
if single.is_empty() || batch.is_empty() {
eprintln!(
"skip: the '{}' goldens hold no min_confidence cases; regenerate them with \
tools/regen_golden.py",
checkpoint.subdir()
);
continue;
}
checked_any = true;
let mut gated_answers = 0;
for case in single {
let root = golden.load_case(&case);
let state = root["state"].clone();
let questions = common::golden_data::build_questions(&root["questions"])
.unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
let options = common::golden_data::recorded_predict_options(&root);
assert!(
options.min_confidence.is_some(),
"{case}: records a threshold"
);
let result = engine
.predict_with_options(state, &questions, &options)
.unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
common::result_compare::assert_result_matches(
&format!("{}/{case}", checkpoint.subdir()),
&root["result"],
&result,
);
gated_answers += result
.iter()
.filter(|(_, answer)| answer.abstention().is_some())
.count();
}
assert!(gated_answers > 0, "no recorded case reported a gate state");
}
if !checked_any {
eprintln!("skip: no ONNX artifacts + min_confidence golden cases found for any checkpoint");
}
}