Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! Shared "does this `LayaResult` match the recorded golden `result`" comparison, factored out of
//! `predict_parity_tests.rs`'s per-case assertions so the Tier 2 routing/shortlist end-to-end
//! tests (which record a plain `result` the same way every other golden case does) can reuse it
//! instead of duplicating the tolerance logic.
use laya::LayaResult;
use laya::answers::{Answer, ChoiceAnswer, NoulAnswer, ScoreAnswer};
use laya::sequence_builder::SequenceBuilder;
use serde_json::Value;
/// Probabilities and confidences: 4-dp recorded values, so a rounding boundary is 1e-4 wide.
pub const PROB_TOLERANCE: f64 = 2e-4;
/// A score is a weighted sum over level indices, so it carries several of those errors.
pub const SCORE_TOLERANCE: f64 = 2e-3;
fn assert_close(expected: f64, actual: f64, tolerance: f64, what: &str) {
let diff = (expected - actual).abs();
assert!(
diff <= tolerance,
"{what}: python {expected}, rust {actual} (diff {diff}, tolerance {tolerance})"
);
}
/// Assert every field of `result` (model, usage, answer ids/types/probabilities/confidence/
/// action/choice/score) matches the recorded golden `expected` (a case's `"result"` field) within
/// the standard parity tolerances. `where_case` prefixes every assertion message.
pub fn assert_result_matches(where_case: &str, expected: &Value, result: &LayaResult) {
assert_state_usage_matches(where_case, expected, result);
assert_eq!(
expected["model"].as_str().unwrap(),
result.model(),
"{where_case}: model"
);
assert_eq!(
expected["usage"]["input_tokens"].as_u64().unwrap() as u32,
result.usage().input_tokens,
"{where_case}: usage.input_tokens"
);
assert_eq!(
expected["usage"]["output_tokens"].as_u64().unwrap() as u32,
result.usage().output_tokens,
"{where_case}: usage.output_tokens"
);
assert_answers_match(where_case, expected, result);
}
/// Assert the answer ids (in order) and every answer's type, confidences, action and per-type
/// probabilities match the golden `expected` result, ignoring its model name and usage.
pub fn assert_answers_match(where_case: &str, expected: &Value, result: &LayaResult) {
let answers = expected["answers"].as_object().unwrap();
let expected_ids: Vec<&str> = answers.keys().map(String::as_str).collect();
assert_eq!(
expected_ids,
result.ids(),
"{where_case}: answer ids in order"
);
for (id, want) in answers {
let got = result
.get(id)
.unwrap_or_else(|| panic!("{where_case}/{id}: missing answer"));
let where_q = format!("{where_case}/{id}");
assert_eq!(
want["type"].as_str().unwrap(),
SequenceBuilder::type_name(got.question_type()),
"{where_q}: type"
);
assert_close(
want["confidence"].as_f64().unwrap(),
got.confidence(),
PROB_TOLERANCE,
&format!("{where_q}.confidence"),
);
assert_close(
want["answer_confidence"].as_f64().unwrap(),
got.answer_confidence(),
PROB_TOLERANCE,
&format!("{where_q}.answer_confidence"),
);
assert_close(
want["action"]["act_probability"].as_f64().unwrap(),
got.action().act_probability,
PROB_TOLERANCE,
&format!("{where_q}.action.act_probability"),
);
assert_gate_matches(want, got, &where_q);
match got {
Answer::Choice(choice) => assert_choice(want, choice, &where_q),
Answer::Score(score) => assert_score(want, score, &where_q),
Answer::Noul(noul) => assert_noul(want, noul, &where_q),
}
}
}
/// What the abstention gate wrote on one answer must be what Python wrote: `low_confidence` only
/// when flagged, `abstention` and `abstention_threshold` only when a threshold was set. A golden
/// recorded without a threshold carries none of the three, so this also pins "unset writes
/// nothing".
pub fn assert_gate_matches(want: &Value, got: &Answer, where_q: &str) {
assert_eq!(
want.get("low_confidence") == Some(&Value::Bool(true)),
got.low_confidence(),
"{where_q}: low_confidence"
);
assert!(
want.get("low_confidence").is_none() || want["low_confidence"] == Value::Bool(true),
"{where_q}: python only ever writes low_confidence: true"
);
assert_eq!(
want.get("abstention").and_then(Value::as_str),
got.abstention(),
"{where_q}: abstention"
);
assert_eq!(
want.get("abstention_threshold").and_then(Value::as_f64),
got.abstention_threshold(),
"{where_q}: abstention_threshold"
);
}
pub fn assert_state_usage_matches(where_case: &str, expected: &Value, result: &LayaResult) {
let usage = result
.state_usage()
.expect("engine result must report state usage");
let want = &expected["usage"];
let options = want.get("options").and_then(Value::as_object);
assert_eq!(
want.as_object().unwrap().len(),
6 + usize::from(options.is_some()),
"{where_case}: new Python usage fields need a Rust port"
);
assert_eq!(
options.map_or(0, |o| o.len()),
usage.options.len(),
"{where_case}: usage.options count"
);
if let Some(options) = options {
for (id, expected) in options {
assert_eq!(
expected.as_object().unwrap().len(),
3,
"{where_case}/{id}: new option usage fields need a Rust port"
);
let got = usage
.options
.get(id)
.expect("collapsed option diagnostic missing");
assert_eq!(
expected["total"].as_u64().unwrap() as usize,
got.total,
"{where_case}/{id}: usage.options.total"
);
assert_eq!(
expected["distinct"].as_u64().unwrap() as usize,
got.distinct,
"{where_case}/{id}: usage.options.distinct"
);
assert_eq!(
expected["tokens_per_option"].as_u64().map(|v| v as usize),
got.tokens_per_option,
"{where_case}/{id}: usage.options.tokens_per_option"
);
}
}
assert_eq!(
want["state_tokens"].as_u64().unwrap() as usize,
usage.state_tokens,
"{where_case}: usage.state_tokens"
);
assert_eq!(
want["state_tokens_dropped"].as_u64().unwrap() as usize,
usage.state_tokens_dropped,
"{where_case}: usage.state_tokens_dropped"
);
assert_eq!(
want["truncated"].as_bool().unwrap(),
usage.truncated,
"{where_case}: usage.truncated"
);
let ids: Vec<&str> = want["truncated_questions"]
.as_array()
.unwrap()
.iter()
.map(|id| id.as_str().unwrap())
.collect();
let got: Vec<&str> = usage
.truncated_questions
.iter()
.map(String::as_str)
.collect();
assert_eq!(ids, got, "{where_case}: usage.truncated_questions");
}
fn assert_choice(want: &Value, got: &ChoiceAnswer, where_q: &str) {
let probabilities = want["probabilities"].as_object().unwrap();
let expected_labels: Vec<&str> = probabilities.keys().map(String::as_str).collect();
let actual_labels: Vec<&str> = got.probabilities.keys().map(String::as_str).collect();
assert_eq!(
expected_labels, actual_labels,
"{where_q}: probability label order"
);
for (label, value) in probabilities {
assert_close(
value.as_f64().unwrap(),
got.probability(label).unwrap(),
PROB_TOLERANCE,
&format!("{where_q}.probabilities[{label}]"),
);
}
assert_eq!(
want["choice"].as_str().unwrap(),
got.choice,
"{where_q}: choice"
);
}
fn assert_score(want: &Value, got: &ScoreAnswer, where_q: &str) {
let legend = want["legend"].as_object().unwrap();
assert_eq!(legend.len(), got.legend.len(), "{where_q}: legend length");
for i in 0..got.legend.len() {
let expected_level = &legend[&i.to_string()];
assert_eq!(expected_level, &got.legend[i], "{where_q}: legend[{i}]");
}
let probabilities = want["probabilities"].as_object().unwrap();
for i in 0..got.probabilities.len() {
assert_close(
probabilities[&i.to_string()].as_f64().unwrap(),
got.probabilities[i],
PROB_TOLERANCE,
&format!("{where_q}.probabilities[{i}]"),
);
}
assert_close(
want["score"].as_f64().unwrap(),
got.score,
SCORE_TOLERANCE,
&format!("{where_q}.score"),
);
}
fn assert_noul(want: &Value, got: &NoulAnswer, where_q: &str) {
assert_close(
want["noul"].as_f64().unwrap(),
got.probability,
PROB_TOLERANCE,
&format!("{where_q}.noul"),
);
}