Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
//! Tier 2: full answers against the ones Python recorded, which needs the model weights.
//!
//! The goldens come from the real `laya.Agent.system_one` running over the same ONNX graph, so any
//! gap here is this port's, not the model's. Recorded values are rounded to four decimals on both
//! sides, which is why the tolerances sit a little above 1e-4 rather than at zero.
//!
//! Run the full suite (all three checkpoints) with:
//! `LAYA_ONNX_ROOT=<parent-of-checkpoints> cargo test --release -- --test-threads=1`
//! `--test-threads=1` bounds memory: each test below loads one ~1.3-1.7 GB engine per checkpoint
//! into a local variable and drops it before moving to the next, so at most one is ever resident
//! *within* a test, but cargo's default parallel test scheduling could otherwise still run two of
//! these `#[test]` functions — each holding its own engine — at the same time.
mod common;
use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
use laya::answers::{Answer, ChoiceAnswer, NoulAnswer, ScoreAnswer};
use laya::error::LayaError;
use laya::laya_options::LayaCheckpoint;
use laya::questions::QuestionSet;
use laya::sequence_builder::SequenceBuilder;
use serde_json::Value;
use std::sync::{Arc, Mutex};
use std::time::Instant;
/// Tracks the worst absolute diff seen across every [`assert_close`] call, purely for the final
/// report's "max observed diff vs golden" figure — not a correctness check by itself.
static MAX_DIFF: Mutex<f64> = Mutex::new(0.0);
/// Probabilities and confidences: 4-dp recorded values, so a rounding boundary is 1e-4 wide.
const PROB_TOLERANCE: f64 = 2e-4;
/// A score is a weighted sum over level indices, so it carries several of those errors.
const SCORE_TOLERANCE: f64 = 2e-3;
fn assert_close(expected: f64, actual: f64, tolerance: f64, what: &str) {
let diff = (expected - actual).abs();
{
let mut max_diff = MAX_DIFF.lock().unwrap();
if diff > *max_diff {
*max_diff = diff;
}
}
assert!(
diff <= tolerance,
"{what}: python {expected}, rust {actual} (diff {diff}, tolerance {tolerance})"
);
}
#[test]
fn reproduces_the_recorded_answers() {
let mut checked_any = false;
for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
eprintln!("skip: no golden data for '{}'", checkpoint.subdir());
continue;
}
for case in golden.success_cases() {
checked_any = true;
let root = golden.load_case(&case);
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"])
.unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
let result = engine
.predict_with_options(
state,
&questions,
&golden_data::recorded_predict_options(&root),
)
.unwrap_or_else(|e| panic!("{}/{case}: predict failed: {e}", checkpoint.subdir()));
let expected = &root["result"];
let where_case = format!("{}/{case}", checkpoint.subdir());
common::result_compare::assert_state_usage_matches(&where_case, expected, &result);
assert_eq!(
expected["model"].as_str().unwrap(),
result.model(),
"{where_case}: model"
);
assert_eq!(
expected["usage"]["input_tokens"].as_u64().unwrap() as u32,
result.usage().input_tokens,
"{where_case}: usage.input_tokens"
);
assert_eq!(
expected["usage"]["output_tokens"].as_u64().unwrap() as u32,
result.usage().output_tokens,
"{where_case}: usage.output_tokens"
);
let answers = expected["answers"].as_object().unwrap();
// Ids in order: a result that answered the right questions in the wrong order would
// attach every answer to the wrong question, and per-id comparison alone would not
// notice.
let expected_ids: Vec<&str> = answers.keys().map(String::as_str).collect();
assert_eq!(
expected_ids,
result.ids(),
"{where_case}: answer ids in order"
);
for (id, want) in answers {
let got = result
.get(id)
.unwrap_or_else(|| panic!("{where_case}/{id}: missing answer"));
let where_q = format!("{where_case}/{id}");
assert_eq!(
want["type"].as_str().unwrap(),
SequenceBuilder::type_name(got.question_type()),
"{where_q}: type"
);
assert_close(
want["confidence"].as_f64().unwrap(),
got.confidence(),
PROB_TOLERANCE,
&format!("{where_q}.confidence"),
);
assert_close(
want["answer_confidence"].as_f64().unwrap(),
got.answer_confidence(),
PROB_TOLERANCE,
&format!("{where_q}.answer_confidence"),
);
assert_close(
want["action"]["act_probability"].as_f64().unwrap(),
got.action().act_probability,
PROB_TOLERANCE,
&format!("{where_q}.action.act_probability"),
);
common::result_compare::assert_gate_matches(want, got, &where_q);
match got {
Answer::Choice(choice) => assert_choice(want, choice, &where_q),
Answer::Score(score) => assert_score(want, score, &where_q),
Answer::Noul(noul) => assert_noul(want, noul, &where_q),
}
}
}
}
if !checked_any {
eprintln!("skip: no golden data + artifacts found for any checkpoint");
} else {
eprintln!(
"reproduces_the_recorded_answers: max abs diff across probability/confidence/action/score assertions = {}",
*MAX_DIFF.lock().unwrap()
);
}
}
#[test]
fn reproduces_the_recorded_answers_for_the_split_layout() {
// Same golden data, same assertions as `reproduces_the_recorded_answers`, but against
// `<repo>/onnx-split/<ckpt>` (the layout laya-ts's exporter produces: `encoder.onnx` +
// `head.onnx`) instead of the fused `model.onnx`. Skips per checkpoint exactly like the fused
// test does when that checkpoint's split-layout export is not present locally — most
// machines will not have one, and that is expected, not a failure.
let mut checked_any = false;
for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load_split(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
eprintln!("skip: no golden data for '{}'", checkpoint.subdir());
continue;
}
for case in golden.success_cases() {
checked_any = true;
let root = golden.load_case(&case);
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"])
.unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
let result = engine
.predict_with_options(
state,
&questions,
&golden_data::recorded_predict_options(&root),
)
.unwrap_or_else(|e| panic!("{}/{case}: predict failed: {e}", checkpoint.subdir()));
let expected = &root["result"];
let where_case = format!("{}/{case} (split layout)", checkpoint.subdir());
common::result_compare::assert_state_usage_matches(&where_case, expected, &result);
assert_eq!(
expected["model"].as_str().unwrap(),
result.model(),
"{where_case}: model"
);
let answers = expected["answers"].as_object().unwrap();
let expected_ids: Vec<&str> = answers.keys().map(String::as_str).collect();
assert_eq!(
expected_ids,
result.ids(),
"{where_case}: answer ids in order"
);
for (id, want) in answers {
let got = result
.get(id)
.unwrap_or_else(|| panic!("{where_case}/{id}: missing answer"));
let where_q = format!("{where_case}/{id}");
assert_close(
want["confidence"].as_f64().unwrap(),
got.confidence(),
PROB_TOLERANCE,
&format!("{where_q}.confidence"),
);
assert_close(
want["answer_confidence"].as_f64().unwrap(),
got.answer_confidence(),
PROB_TOLERANCE,
&format!("{where_q}.answer_confidence"),
);
common::result_compare::assert_gate_matches(want, got, &where_q);
match got {
Answer::Choice(choice) => assert_choice(want, choice, &where_q),
Answer::Score(score) => assert_score(want, score, &where_q),
Answer::Noul(noul) => assert_noul(want, noul, &where_q),
}
}
}
}
if !checked_any {
eprintln!(
"skip: no split-layout ONNX artifacts + golden data found for any checkpoint under \
<repo>/onnx-split/"
);
}
}
#[test]
fn rejects_the_same_questions_python_rejects() {
let mut checked_any = false;
for checkpoint in engines::checkpoints() {
let Some(engine) = engines::load(checkpoint) else {
continue;
};
let golden = CheckpointGoldenData::for_checkpoint(checkpoint.subdir());
if !golden.available() {
continue;
}
for case in golden.error_cases() {
checked_any = true;
let root = golden.load_case(&case);
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"])
.unwrap_or_else(|e| panic!("{}/{case}: {e}", checkpoint.subdir()));
let error = engine
.predict_with_options(
state,
&questions,
&golden_data::recorded_predict_options(&root),
)
.expect_err(&format!(
"{}/{case}: expected predict to fail",
checkpoint.subdir()
));
let expected_message = root["error_message"].as_str().unwrap();
let actual_message = error.to_string();
// A rejected threshold is Python's ValueError text, whole.
if root
.get("options")
.is_some_and(|options| options.get("min_confidence").is_some())
{
assert!(
matches!(error, LayaError::InvalidMinConfidence(_)),
"{}/{case}: expected LayaError::InvalidMinConfidence, got {error:?}",
checkpoint.subdir()
);
assert_eq!(
expected_message,
actual_message,
"{}/{case}: error message",
checkpoint.subdir()
);
continue;
}
assert!(
actual_message.starts_with(expected_message),
"{}/{case}: error message mismatch\n python: {expected_message}\n rust: {actual_message}",
checkpoint.subdir()
);
// The specific variant matters too: this failure mode is always a marker/head-budget
// problem, never e.g. a missing question id.
assert!(
matches!(error, LayaError::NoMarkers { .. }),
"{}/{case}: expected LayaError::NoMarkers, got {error:?}",
checkpoint.subdir()
);
}
}
if !checked_any {
eprintln!("skip: no error-case golden data + artifacts found for any checkpoint");
}
}
#[test]
fn answers_the_same_way_when_questions_are_asked_one_at_a_time() {
// Every question rides in one batch, so a padding or masking mistake could let a question be
// influenced by its neighbours. Asking them singly has to give the same answers.
let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
if !golden.available() {
eprintln!("skip: no golden data for 'multilingual'");
return;
}
let root = golden.load("case_quickstart.json");
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"]).expect("build_questions");
let batched = engine
.predict(state.clone(), &questions)
.expect("batched predict");
for id in questions.ids() {
let single_set = QuestionSet::new()
.with(id, questions.get(id).unwrap().clone())
.expect("single-question set");
let single = engine
.predict(state.clone(), &single_set)
.expect("single predict");
assert_close(
batched[id].confidence(),
single[id].confidence(),
PROB_TOLERANCE,
&format!("{id}.confidence"),
);
}
}
#[test]
fn is_safe_to_call_from_several_threads_at_once() {
// The engine is documented as shareable and a caller will hold it as a singleton, so the ONNX
// session mutex and the tokenizer both have to tolerate concurrent use.
let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
if !golden.available() {
eprintln!("skip: no golden data for 'multilingual'");
return;
}
let root = golden.load("case_quickstart.json");
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"]).expect("build_questions");
let engine = Arc::new(engine);
let want = engine
.predict(state.clone(), &questions)
.expect("warm-up predict")
.get("department")
.unwrap()
.as_choice()
.unwrap()
.choice
.clone();
let handles: Vec<_> = (0..8)
.map(|_| {
let engine = Arc::clone(&engine);
let state = state.clone();
let questions = questions.clone();
std::thread::spawn(move || {
engine
.predict(state, &questions)
.expect("concurrent predict")
.get("department")
.unwrap()
.as_choice()
.unwrap()
.choice
.clone()
})
})
.collect();
for handle in handles {
let choice = handle.join().expect("thread panicked");
assert_eq!(want, choice);
}
}
#[test]
fn warm_predict_latency_is_reported() {
// Not a parity check: reports the warm (post-warm-up) predict latency for the final report,
// the way samples/Laya.Sample's Program.cs does with a Stopwatch. Run with `--nocapture` to
// see the printed line.
let Some(engine) = engines::load_selected(LayaCheckpoint::Multilingual) else {
return;
};
let golden = CheckpointGoldenData::for_checkpoint(LayaCheckpoint::Multilingual.subdir());
if !golden.available() {
eprintln!("skip: no golden data for 'multilingual'");
return;
}
let root = golden.load("case_quickstart.json");
let state = root["state"].clone();
let questions = golden_data::build_questions(&root["questions"]).expect("build_questions");
// First call pays for ONNX Runtime session warm-up.
let cold_start = Instant::now();
engine
.predict(state.clone(), &questions)
.expect("cold predict");
let cold = cold_start.elapsed();
let warm_start = Instant::now();
engine.predict(state, &questions).expect("warm predict");
let warm = warm_start.elapsed();
eprintln!(
"warm_predict_latency_is_reported: multilingual cold={cold:?} warm={warm:?} (quickstart, {} questions)",
questions.len()
);
}
// ── helpers ───────────────────────────────────────────────────────────────
fn assert_choice(want: &Value, got: &ChoiceAnswer, where_q: &str) {
let probabilities = want["probabilities"].as_object().unwrap();
// Labels in order, because the entire marker layout rests on labels being positional.
let expected_labels: Vec<&str> = probabilities.keys().map(String::as_str).collect();
let actual_labels: Vec<&str> = got.probabilities.keys().map(String::as_str).collect();
assert_eq!(
expected_labels, actual_labels,
"{where_q}: probability label order"
);
for (label, value) in probabilities {
assert_close(
value.as_f64().unwrap(),
got.probability(label).unwrap(),
PROB_TOLERANCE,
&format!("{where_q}.probabilities[{label}]"),
);
}
// Checked after the distribution: when the top two are close the argmax is the fragile part,
// and the distribution mismatch is the more informative failure to report.
assert_eq!(
want["choice"].as_str().unwrap(),
got.choice,
"{where_q}: choice"
);
let sum: f64 = got.probabilities.values().sum();
assert_close(1.0, sum, 1e-3, &format!("{where_q}: probabilities sum"));
}
fn assert_score(want: &Value, got: &ScoreAnswer, where_q: &str) {
// Python's JSON keys both the legend and the distribution by the stringified level index.
let legend = want["legend"].as_object().unwrap();
assert_eq!(legend.len(), got.legend.len(), "{where_q}: legend length");
for i in 0..got.legend.len() {
let expected_level = &legend[&i.to_string()];
assert_eq!(expected_level, &got.legend[i], "{where_q}: legend[{i}]");
}
let probabilities = want["probabilities"].as_object().unwrap();
assert_eq!(
probabilities.len(),
got.probabilities.len(),
"{where_q}: probabilities length"
);
for i in 0..got.probabilities.len() {
assert_close(
probabilities[&i.to_string()].as_f64().unwrap(),
got.probabilities[i],
PROB_TOLERANCE,
&format!("{where_q}.probabilities[{i}]"),
);
}
assert_close(
want["score"].as_f64().unwrap(),
got.score,
SCORE_TOLERANCE,
&format!("{where_q}.score"),
);
}
fn assert_noul(want: &Value, got: &NoulAnswer, where_q: &str) {
assert_close(
want["noul"].as_f64().unwrap(),
got.probability,
PROB_TOLERANCE,
&format!("{where_q}.noul"),
);
}