Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! The abstention gate: per-call `min_confidence`, the states it reports, and Jev's `confidence`.
//! Ports `laya/confidence.py` (#361, #394), function for function.
//!
//! Python hands the gate a plain `float` or a `dict` and validates it with
//! `check_min_confidence`. Here a threshold is a [`MinConfidence`], built through
//! [`MinConfidence::scalar`] or [`MinConfidence::map`], which raise Python's own `ValueError`
//! text as [`LayaError::InvalidMinConfidence`]. The variants are public, so an unvalidated value
//! can be written by hand; [`MinConfidence::validate`] therefore runs again where Python runs
//! `check_min_confidence` (the start of `predict_batch`, and the router's `predict` and
//! `predict_batch`), so a hand-built value cannot reach the gate.
//!
//! What cannot be reached from Rust is the type check (`bool`, `str`, `None` in place of a float)
//! and the `isinstance(m, dict)` check: the types already rule them out.
use crate::answers::{Answer, LayaResult};
use crate::error::{LayaError, Result};
use crate::laya_router::python_repr_str;
use crate::python_json::PythonJson;
use indexmap::IndexMap;
/// The qtype half of a bucket key, in `common.temp_bucket`'s spelling.
pub const BUCKET_QTYPES: [&str; 3] = ["choice", "score", "noul"];
/// The option-count half of a bucket key.
pub const BUCKET_SIZES: [&str; 4] = ["2", "3-5", "6-10", "11+"];
/// The one non-bucket key a map may carry: the threshold for every bucket the map does not name.
pub const DEFAULT_BUCKET_KEY: &str = "default";
/// A gate ran and this answer's confidence cleared it.
pub const GATE_PASSED: &str = "passed";
/// A gate ran and this answer's confidence fell below it.
pub const GATE_ABSTAINED: &str = "abstained";
/// A gate ran and this answer carried no usable confidence.
pub const GATE_UNEVALUATED: &str = "unevaluated";
/// The states [`Answer::abstention`] reports, and the only ones. There is deliberately no member
/// for "no gate was configured": with no `min_confidence` an answer reports nothing at all
/// ([`Answer::abstention`] is `None`).
pub const GATE_STATES: [&str; 3] = [GATE_PASSED, GATE_ABSTAINED, GATE_UNEVALUATED];
/// Every option-count bucket a `min_confidence` map can name, e.g. `"choice:3-5"`. Ports
/// `bucket_keys`. Python returns a set; this is the same twelve keys in qtype-major order.
pub fn bucket_keys() -> Vec<String> {
BUCKET_QTYPES
.iter()
.flat_map(|qtype| {
BUCKET_SIZES
.iter()
.map(move |size| format!("{qtype}:{size}"))
})
.collect()
}
/// Python's `repr(float)`, which spells the non-finite values `nan`, `inf` and `-inf` (unlike
/// the JSON spelling [`PythonJson::repr`] gives them).
fn repr_float(value: f64) -> String {
if value.is_nan() {
"nan".to_string()
} else if value.is_infinite() {
if value > 0.0 { "inf" } else { "-inf" }.to_string()
} else {
PythonJson::repr(value)
}
}
/// Python's `repr(dict)` for a `str -> float` map: `{'choice:2': 1.5, 'default': 0.5}`.
fn repr_map(map: &IndexMap<String, f64>) -> String {
let items: Vec<String> = map
.iter()
.map(|(key, value)| format!("{}: {}", python_repr_str(key), repr_float(*value)))
.collect();
format!("{{{}}}", items.join(", "))
}
/// Ports `_check_one_threshold`.
fn check_one_threshold(value: f64) -> Result<f64> {
if !value.is_finite() || !(0.0..=1.0).contains(&value) {
return Err(LayaError::InvalidMinConfidence(format!(
"min_confidence must be a float in [0.0, 1.0], got {}",
repr_float(value)
)));
}
Ok(value)
}
/// An opt-in abstention threshold on `answer_confidence`: one number for every answer, or a
/// per-bucket map so the threshold can differ by option count (#394).
///
/// Build one with [`Self::scalar`] or [`Self::map`], which validate as Python's
/// `check_min_confidence` does.
#[derive(Debug, Clone, PartialEq)]
pub enum MinConfidence {
/// One threshold for every answer, in `[0.0, 1.0]`.
Scalar(f64),
/// A threshold per option-count bucket (every [`bucket_keys`] member) plus the optional
/// `"default"` for any bucket the map does not name.
Map(IndexMap<String, f64>),
}
impl MinConfidence {
/// A validated scalar threshold. Errors with Python's text, e.g.
/// `min_confidence must be a float in [0.0, 1.0], got 1.5`.
pub fn scalar(threshold: f64) -> Result<Self> {
let value = Self::Scalar(threshold);
value.validate()?;
Ok(value)
}
/// A validated per-bucket map, in the order given (which decides which entry an error names
/// first). Keys are [`bucket_keys`] members or `"default"`; values are floats in `[0.0, 1.0]`.
pub fn map<K: Into<String>>(thresholds: impl IntoIterator<Item = (K, f64)>) -> Result<Self> {
let value = Self::Map(
thresholds
.into_iter()
.map(|(key, threshold)| (key.into(), threshold))
.collect(),
);
value.validate()?;
Ok(value)
}
/// Ports `check_min_confidence` / `check_min_confidence_map`. Run again wherever Python runs
/// it, because the variants are public.
pub fn validate(&self) -> Result<()> {
match self {
Self::Scalar(threshold) => check_one_threshold(*threshold).map(|_| ()),
Self::Map(map) => {
if map.is_empty() {
return Err(LayaError::InvalidMinConfidence(format!(
"a min_confidence map must be a non-empty dict of bucket -> float, got {}",
repr_map(map)
)));
}
let allowed = bucket_keys();
for (key, threshold) in map {
if key != DEFAULT_BUCKET_KEY && !allowed.contains(key) {
return Err(LayaError::InvalidMinConfidence(format!(
"min_confidence map keys must be a bucket like 'choice:3-5' or \
'default', got {}",
python_repr_str(key)
)));
}
check_one_threshold(*threshold)?;
}
Ok(())
}
}
}
/// Ports `resolve_min_confidence` (a scalar passes straight through): the threshold this
/// answer is gated at.
fn resolve(&self, answer: &Answer) -> f64 {
match self {
Self::Scalar(threshold) => *threshold,
Self::Map(map) => answer
.option_bucket()
.and_then(|key| map.get(&key))
.or_else(|| map.get(DEFAULT_BUCKET_KEY))
.copied()
.unwrap_or(0.0),
}
}
}
/// The gate's verdict on one answer, kept on the answer.
#[derive(Debug, Clone, Copy, PartialEq, Default)]
pub(crate) struct Gate {
pub(crate) low_confidence: bool,
pub(crate) abstention: Option<&'static str>,
pub(crate) threshold: Option<f64>,
}
/// Ports `_gate_confidence`: `answer_confidence` when usable, else `confidence`, else nothing.
fn gate_confidence(answer: &Answer) -> Option<f64> {
[answer.answer_confidence(), answer.confidence()]
.into_iter()
.find(|value| value.is_finite())
}
/// Ports `flag_low_confidence` and `apply_confidence_gate` for one answer. The verdict replaces
/// whatever an earlier gate left, so a reused answer cannot keep a stale flag (#910).
fn gate_answer(answer: &mut Answer, min_confidence: &MinConfidence) {
let threshold = min_confidence.resolve(answer);
let confidence = gate_confidence(answer);
let low_confidence = confidence.is_some_and(|c| c < threshold);
let abstention = if low_confidence {
GATE_ABSTAINED
} else if confidence.is_none() {
GATE_UNEVALUATED
} else {
GATE_PASSED
};
answer.set_gate(Gate {
low_confidence,
abstention: Some(abstention),
threshold: Some(threshold),
});
}
/// Ports `apply_confidence_gate`: with a threshold, reports the gate's state on every answer of
/// every result; with `None`, writes nothing.
pub(crate) fn apply_confidence_gate(
results: &mut [LayaResult],
min_confidence: Option<&MinConfidence>,
) {
let Some(min_confidence) = min_confidence else {
return;
};
for result in results {
result.for_each_answer_mut(|answer| gate_answer(answer, min_confidence));
}
}
/// Python's `sum()` over floats. Since CPython 3.12 it is not a plain left-to-right addition: it
/// carries a Neumaier compensation, so `sum([0.7, 0.2, 0.1])` is `1.0` where the naive loop gives
/// `0.9999999999999999`. This is that algorithm (`builtin_sum_impl`, float branch), so the totals
/// and spreads below round as Python 3.12+ rounds them.
fn python_sum(values: impl IntoIterator<Item = f64>) -> f64 {
let mut values = values.into_iter();
// The start value is the int 0; adding the first float gives that float.
let mut sum = match values.next() {
Some(first) => 0.0 + first,
None => return 0.0,
};
let mut compensation = 0.0_f64;
for value in values {
let total = sum + value;
if sum.abs() >= value.abs() {
compensation += (sum - total) + value;
} else {
compensation += (value - total) + sum;
}
sum = total;
}
// Do not let the compensation turn an infinite or overflowed sum into NaN.
if compensation != 0.0 && compensation.is_finite() {
sum += compensation;
}
sum
}
/// Ports `jev_confidence` over an answer's (already rounded) probabilities: the `confidence` Jev
/// would report, or `None` for a noul answer and for a malformed document.
pub(crate) fn jev_confidence(answer: &Answer) -> Option<f64> {
let probabilities: Vec<f64> = match answer {
Answer::Choice(choice) => choice.probabilities.values().copied().collect(),
Answer::Score(score) => score.probabilities.clone(),
Answer::Noul(_) => return None,
};
if probabilities.is_empty() || !probabilities.iter().all(|v| v.is_finite() && *v >= 0.0) {
return None;
}
let n = probabilities.len();
if n == 1 {
return Some(1.0);
}
let count = n as f64;
let total = python_sum(probabilities.iter().copied());
let p: Vec<f64> = if total != 0.0 {
probabilities.iter().map(|v| v / total).collect()
} else {
vec![1.0 / count; n]
};
// `max` and `max(range(n), key=...)` keep the first of equal values.
let mut mode = 0;
for (i, value) in p.iter().enumerate().skip(1) {
if *value > p[mode] {
mode = i;
}
}
if matches!(answer, Answer::Choice(_)) {
return Some((p[mode] - 1.0 / count) / (1.0 - 1.0 / count));
}
let spread = python_sum(
p.iter()
.enumerate()
.map(|(i, v)| v * (i as f64 - mode as f64).abs()),
);
let center = (n - 1) as f64 / 2.0;
let uniform_spread = python_sum((0..n).map(|i| (i as f64 - center).abs())) / count;
// Python's `max(0.0, x)`: zero unless `x` is greater.
let jev = 1.0 - spread / uniform_spread;
Some(if jev > 0.0 { jev } else { 0.0 })
}
/// Ports `_option_bucket`: the `qtype:size` key an answer is gated under.
pub(crate) fn option_bucket(answer: &Answer) -> Option<String> {
let (qtype, options) = match answer {
Answer::Choice(choice) => ("choice", choice.probabilities.len()),
Answer::Score(score) => ("score", score.probabilities.len()),
// A noul answer carries no probabilities dict, so Python reads it as two options.
Answer::Noul(_) => ("noul", 2),
};
if options == 0 {
return None;
}
let size = match options {
0..=2 => "2",
3..=5 => "3-5",
6..=10 => "6-10",
_ => "11+",
};
Some(format!("{qtype}:{size}"))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::answers::{ActionInfo, ChoiceAnswer, NoulAnswer, ScoreAnswer};
use std::collections::HashMap;
fn choice(probabilities: &[f64], confidence: f64, answer_confidence: f64) -> Answer {
let map: IndexMap<String, f64> = probabilities
.iter()
.enumerate()
.map(|(i, p)| (format!("o{i}"), *p))
.collect();
Answer::Choice(ChoiceAnswer::new(
"o0",
map,
confidence,
answer_confidence,
ActionInfo::default(),
))
}
fn score(probabilities: &[f64]) -> Answer {
Answer::Score(ScoreAnswer::new(
0.0,
Vec::new(),
probabilities.to_vec(),
0.5,
0.5,
ActionInfo::default(),
))
}
fn noul(confidence: f64, answer_confidence: f64) -> Answer {
Answer::Noul(NoulAnswer::new(
0.7,
confidence,
answer_confidence,
ActionInfo::default(),
))
}
fn result(answers: Vec<(&str, Answer)>) -> LayaResult {
let order = answers.iter().map(|(id, _)| id.to_string()).collect();
let by_id: HashMap<String, Answer> = answers
.into_iter()
.map(|(id, a)| (id.to_string(), a))
.collect();
LayaResult::new("m", order, by_id, crate::answers::Usage::default())
}
fn gated(answer: Answer, min_confidence: &MinConfidence) -> Answer {
let mut results = [result(vec![("q", answer)])];
apply_confidence_gate(&mut results, Some(min_confidence));
results[0]["q"].clone()
}
#[test]
fn an_ungated_answer_reports_nothing() {
let answer = choice(&[0.6, 0.4], 0.3, 0.6);
assert!(!answer.low_confidence());
assert_eq!(answer.abstention(), None);
assert_eq!(answer.abstention_threshold(), None);
let mut results = [result(vec![("q", answer)])];
apply_confidence_gate(&mut results, None);
assert_eq!(results[0]["q"].abstention(), None);
}
#[test]
fn scalar_gate_reports_passed_abstained_and_the_threshold() {
let threshold = MinConfidence::scalar(0.7).unwrap();
let low = gated(choice(&[0.6, 0.4], 0.9, 0.6), &threshold);
assert!(low.low_confidence());
assert_eq!(low.abstention(), Some("abstained"));
assert_eq!(low.abstention_threshold(), Some(0.7));
let high = gated(choice(&[0.8, 0.2], 0.1, 0.8), &threshold);
assert!(!high.low_confidence());
assert_eq!(high.abstention(), Some("passed"));
// Equal to the threshold clears it: the test is strictly below.
let equal = gated(choice(&[0.7, 0.3], 0.1, 0.7), &threshold);
assert_eq!(equal.abstention(), Some("passed"));
}
#[test]
fn the_gate_reads_answer_confidence_first_then_confidence() {
let threshold = MinConfidence::scalar(0.5).unwrap();
// answer_confidence passes although confidence would not.
let a = gated(choice(&[0.6, 0.4], 0.1, 0.6), &threshold);
assert_eq!(a.abstention(), Some("passed"));
// A NaN answer_confidence falls back to confidence (confidence.py:86).
let b = gated(choice(&[0.6, 0.4], 0.1, f64::NAN), &threshold);
assert_eq!(b.abstention(), Some("abstained"));
let c = gated(choice(&[0.6, 0.4], 0.9, f64::INFINITY), &threshold);
assert_eq!(c.abstention(), Some("passed"));
}
#[test]
fn an_answer_with_no_usable_confidence_is_unevaluated_not_flagged() {
let threshold = MinConfidence::scalar(0.9).unwrap();
let answer = gated(noul(f64::NAN, f64::NAN), &threshold);
assert!(!answer.low_confidence());
assert_eq!(answer.abstention(), Some("unevaluated"));
assert_eq!(answer.abstention_threshold(), Some(0.9));
}
#[test]
fn zero_is_a_gate_that_passes_everything_usable_and_echoes_zero() {
let threshold = MinConfidence::scalar(0.0).unwrap();
let answer = gated(choice(&[0.5, 0.5], 0.0, 0.0), &threshold);
assert_eq!(answer.abstention(), Some("passed"));
assert_eq!(answer.abstention_threshold(), Some(0.0));
assert!(!answer.low_confidence());
let unusable = gated(noul(f64::NAN, f64::NAN), &threshold);
assert_eq!(unusable.abstention(), Some("unevaluated"));
}
#[test]
fn a_map_gates_each_answer_at_its_own_bucket_then_default_then_zero() {
let map =
MinConfidence::map([("choice:2", 0.9), ("score:3-5", 0.8), ("default", 0.55)]).unwrap();
let two = gated(choice(&[0.6, 0.4], 0.5, 0.6), &map);
assert_eq!(two.abstention_threshold(), Some(0.9));
assert_eq!(two.abstention(), Some("abstained"));
let three_scores = gated(score(&[0.2, 0.7, 0.1]), &map);
assert_eq!(three_scores.abstention_threshold(), Some(0.8));
// noul:2 is not named, so it falls to the default.
let unnamed = gated(noul(0.7, 0.7), &map);
assert_eq!(unnamed.abstention_threshold(), Some(0.55));
assert_eq!(unnamed.abstention(), Some("passed"));
// No default either: gate nothing.
let no_default = MinConfidence::map([("choice:2", 0.9)]).unwrap();
let ungated = gated(noul(0.1, 0.1), &no_default);
assert_eq!(ungated.abstention_threshold(), Some(0.0));
assert_eq!(ungated.abstention(), Some("passed"));
}
#[test]
fn option_buckets_follow_the_size_bands() {
let sizes = [
(1, "2"),
(2, "2"),
(3, "3-5"),
(5, "3-5"),
(6, "6-10"),
(10, "6-10"),
(11, "11+"),
];
for (count, size) in sizes {
let p = vec![1.0 / count as f64; count];
assert_eq!(
choice(&p, 0.5, 0.5).option_bucket(),
Some(format!("choice:{size}"))
);
assert_eq!(score(&p).option_bucket(), Some(format!("score:{size}")));
}
assert_eq!(noul(0.5, 0.5).option_bucket().as_deref(), Some("noul:2"));
assert_eq!(choice(&[], 0.5, 0.5).option_bucket(), None);
}
/// #910: a result gated at a strict threshold and then at a lenient one must not keep the
/// flag. Results are values here, so the verdict is rewritten on every gate.
#[test]
fn regating_a_reused_result_clears_a_stale_flag() {
let mut results = [result(vec![("q", choice(&[0.6, 0.4], 0.5, 0.6))])];
apply_confidence_gate(&mut results, Some(&MinConfidence::scalar(0.9).unwrap()));
assert!(results[0]["q"].low_confidence());
apply_confidence_gate(&mut results, Some(&MinConfidence::scalar(0.0).unwrap()));
assert!(!results[0]["q"].low_confidence());
assert_eq!(results[0]["q"].abstention(), Some("passed"));
assert_eq!(results[0]["q"].abstention_threshold(), Some(0.0));
// A later call with no threshold writes nothing, so the last verdict is untouched.
apply_confidence_gate(&mut results, None);
assert_eq!(results[0]["q"].abstention(), Some("passed"));
}
fn approx(a: Option<f64>, b: f64) {
let a = a.expect("a jev confidence");
assert!((a - b).abs() < 1e-12, "{a} vs {b}");
}
#[test]
fn jev_confidence_for_choice_and_score() {
approx(choice(&[0.5, 0.5], 0.0, 0.5).jev_confidence(), 0.0);
approx(choice(&[1.0, 0.0], 0.0, 1.0).jev_confidence(), 1.0);
approx(
choice(&[0.6, 0.3, 0.1], 0.0, 0.6).jev_confidence(),
(0.6 - 1.0 / 3.0) / (2.0 / 3.0),
);
// A single option is 1.0, and a zero total reads as uniform.
approx(choice(&[0.0], 0.0, 0.0).jev_confidence(), 1.0);
approx(choice(&[0.0, 0.0], 0.0, 0.0).jev_confidence(), 0.0);
// Unnormalised input is normalised by its sum.
approx(choice(&[3.0, 1.0], 0.0, 0.0).jev_confidence(), 0.5);
// Score: a point mass has zero spread; a uniform distribution is 0.
approx(score(&[0.0, 1.0, 0.0]).jev_confidence(), 1.0);
approx(score(&[0.25, 0.25, 0.25, 0.25]).jev_confidence(), 0.0);
}
#[test]
fn jev_confidence_is_none_for_noul_and_malformed_probabilities() {
assert_eq!(noul(0.5, 0.5).jev_confidence(), None);
assert_eq!(choice(&[], 0.5, 0.5).jev_confidence(), None);
assert_eq!(choice(&[0.5, f64::NAN], 0.5, 0.5).jev_confidence(), None);
assert_eq!(
choice(&[0.5, f64::INFINITY], 0.5, 0.5).jev_confidence(),
None
);
assert_eq!(choice(&[0.5, -0.1], 0.5, 0.5).jev_confidence(), None);
assert_eq!(score(&[0.5, -0.1]).jev_confidence(), None);
assert_eq!(score(&[]).jev_confidence(), None);
}
#[test]
fn scalar_validation_uses_pythons_text_and_repr() {
let text = |v: f64| MinConfidence::scalar(v).unwrap_err().to_string();
let head = "min_confidence must be a float in [0.0, 1.0], got ";
assert_eq!(text(1.5), format!("{head}1.5"));
assert_eq!(text(2.0), format!("{head}2.0"));
assert_eq!(text(-0.1), format!("{head}-0.1"));
assert_eq!(text(f64::NAN), format!("{head}nan"));
assert_eq!(text(f64::INFINITY), format!("{head}inf"));
assert_eq!(text(f64::NEG_INFINITY), format!("{head}-inf"));
assert_eq!(text(1e16), format!("{head}1e+16"));
assert!(MinConfidence::scalar(0.0).is_ok());
assert!(MinConfidence::scalar(1.0).is_ok());
}
#[test]
fn map_validation_uses_pythons_text_in_python_order() {
let err = |m: Vec<(&str, f64)>| MinConfidence::map(m).unwrap_err().to_string();
let keys = "min_confidence map keys must be a bucket like 'choice:3-5' or 'default', got ";
let value = "min_confidence must be a float in [0.0, 1.0], got 1.5";
assert_eq!(
err(vec![]),
"a min_confidence map must be a non-empty dict of bucket -> float, got {}"
);
assert_eq!(err(vec![("choice:1", 0.5)]), format!("{keys}'choice:1'"));
assert_eq!(err(vec![("it's", 0.5)]), format!("{keys}\"it's\""));
assert_eq!(err(vec![("choice:2", 1.5)]), value);
// The first bad entry in insertion order decides which error is raised.
assert_eq!(err(vec![("choice:2", 1.5), ("nope", 0.5)]), value);
assert_eq!(
err(vec![("nope", 0.5), ("choice:2", 1.5)]),
format!("{keys}'nope'")
);
assert!(MinConfidence::map([("default", 0.5), ("noul:11+", 0.0)]).is_ok());
}
#[test]
fn hand_built_variants_are_validated_again() {
assert!(MinConfidence::Scalar(2.0).validate().is_err());
assert!(MinConfidence::Map(IndexMap::new()).validate().is_err());
}
/// CPython 3.12 compensates `sum()`: the naive loop would give 0.9999999999999999.
#[test]
fn python_sum_is_compensated_like_cpython_312() {
assert_eq!(python_sum([0.7, 0.2, 0.1]), 1.0);
assert_eq!(python_sum([0.1; 10]), 1.0);
assert_eq!(python_sum([]), 0.0);
assert_eq!(python_sum([f64::INFINITY, 1.0]), f64::INFINITY);
}
#[test]
fn bucket_keys_are_the_twelve_buckets() {
let keys = bucket_keys();
assert_eq!(keys.len(), 12);
assert_eq!(keys[0], "choice:2");
assert_eq!(keys[11], "noul:11+");
assert_eq!(GATE_STATES, ["passed", "abstained", "unevaluated"]);
}
}