Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(rust): add calibration and typed answers

kevin-gatimu7b5984bfa50f
diff --git a/laya-rust/src/answers.rs b/laya-rust/src/answers.rs
new file mode 100644
index 00000000..bc909276
--- /dev/null
+++ b/laya-rust/src/answers.rs
@@ -0,0 +1,330 @@
+//! The typed result of one `predict` call. Pure data — nothing here runs the model; `engine.rs`
+//! (a later phase) is what builds these from calibrated probabilities.
+
+use crate::laya_router::RouteDecision;
+use crate::laya_shortlist::ShortlistInfo;
+use crate::questions::QuestionType;
+use indexmap::IndexMap;
+use serde_json::Value;
+use std::collections::HashMap;
+
+/// Token accounting for one `predict` call. Laya emits no tokens, so output is always 0.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct Usage {
+ /// Non-padding tokens across every question in the batch.
+ pub input_tokens: u32,
+ /// Always 0: a System 1 pass generates nothing. Kept for API symmetry with token-based SDKs.
+ pub output_tokens: u32,
+}
+
+/// The action head's read on whether the answer can be acted on directly.
+#[derive(Debug, Clone, Copy, PartialEq, Default)]
+pub struct ActionInfo {
+ /// Probability of acting rather than escalating: softmax over the action logits, first
+ /// component.
+ pub act_probability: f64,
+}
+
+/// The answer to a [`QuestionType::Choice`] question.
+#[derive(Debug, Clone)]
+pub struct ChoiceAnswer {
+ /// The highest-probability label.
+ pub choice: String,
+ /// Probability per label, in the order the options were declared. Rounded to 4 decimals.
+ pub probabilities: IndexMap<String, f64>,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+}
+
+impl ChoiceAnswer {
+ /// Build a choice answer. `probabilities` must be non-empty and in label order.
+ pub fn new(
+ choice: impl Into<String>,
+ probabilities: IndexMap<String, f64>,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+ ) -> Self {
+ Self {
+ choice: choice.into(),
+ probabilities,
+ confidence,
+ answer_confidence,
+ action,
+ }
+ }
+
+ /// The probability assigned to one label, if any option carries that label.
+ pub fn probability(&self, label: &str) -> Option<f64> {
+ self.probabilities.get(label).copied()
+ }
+}
+
+/// The answer to a [`QuestionType::Score`] question.
+#[derive(Debug, Clone)]
+pub struct ScoreAnswer {
+ /// Expected level: the probability-weighted mean of the level indices, a fractional value
+ /// between 0 and `legend.len() - 1`. Rounded to 4 decimals.
+ pub score: f64,
+ /// What each level index means, as supplied on the question.
+ pub legend: Vec<Value>,
+ /// Probability per level index. Rounded to 4 decimals.
+ pub probabilities: Vec<f64>,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+}
+
+impl ScoreAnswer {
+ /// Build a score answer.
+ pub fn new(
+ score: f64,
+ legend: Vec<Value>,
+ probabilities: Vec<f64>,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+ ) -> Self {
+ Self {
+ score,
+ legend,
+ probabilities,
+ confidence,
+ answer_confidence,
+ action,
+ }
+ }
+
+ /// The single most likely level index, as opposed to the expected value. Ties go to the
+ /// lowest index.
+ pub fn most_likely_level(&self) -> usize {
+ let mut best = 0;
+ for (i, &p) in self.probabilities.iter().enumerate().skip(1) {
+ if p > self.probabilities[best] {
+ best = i;
+ }
+ }
+ best
+ }
+}
+
+/// The answer to a [`QuestionType::Noul`] question.
+#[derive(Debug, Clone, Copy)]
+pub struct NoulAnswer {
+ /// Probability that the statement holds, in `[0, 1]`. Rounded to 4 decimals.
+ pub probability: f64,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+}
+
+impl NoulAnswer {
+ /// Build a noul answer.
+ pub fn new(
+ probability: f64,
+ confidence: f64,
+ answer_confidence: f64,
+ action: ActionInfo,
+ ) -> Self {
+ Self {
+ probability,
+ confidence,
+ answer_confidence,
+ action,
+ }
+ }
+
+ /// Whether the statement more likely holds than not.
+ pub fn value(&self) -> bool {
+ self.probability >= 0.5
+ }
+}
+
+/// One answer. Cast with [`Answer::as_choice`], [`Answer::as_score`] or [`Answer::as_noul`].
+#[derive(Debug, Clone)]
+pub enum Answer {
+ /// See [`ChoiceAnswer`].
+ Choice(ChoiceAnswer),
+ /// See [`ScoreAnswer`].
+ Score(ScoreAnswer),
+ /// See [`NoulAnswer`].
+ Noul(NoulAnswer),
+}
+
+impl Answer {
+ /// Which question type produced this answer.
+ pub fn question_type(&self) -> QuestionType {
+ match self {
+ Answer::Choice(_) => QuestionType::Choice,
+ Answer::Score(_) => QuestionType::Score,
+ Answer::Noul(_) => QuestionType::Noul,
+ }
+ }
+
+ /// Calibrated confidence in `[0, 1]`. For choice and score this is normalized Shannon entropy
+ /// (`1 - H(p)/log k`); for noul it is `max(p, 1 - p)`. Rounded to 4 decimals.
+ pub fn confidence(&self) -> f64 {
+ match self {
+ Answer::Choice(c) => c.confidence,
+ Answer::Score(s) => s.confidence,
+ Answer::Noul(n) => n.confidence,
+ }
+ }
+
+ /// Calibrated confidence in `[0, 1]`, on the scale temperature scaling actually fits: `max(p)`
+ /// over the option probabilities, clipped to `[0, 1]` and rounded to 4 decimals. Unlike
+ /// [`Self::confidence`] (normalized entropy for choice/score, `max(p, 1-p)` for noul), this is
+ /// the same quantity for every question type, so a caller can gate across types on one number.
+ /// Ports `answer_confidence` from `laya/common.py` (laya 0.3.21); every ECE figure in the
+ /// Python repository is computed on this quantity, not [`Self::confidence`].
+ pub fn answer_confidence(&self) -> f64 {
+ match self {
+ Answer::Choice(c) => c.answer_confidence,
+ Answer::Score(s) => s.answer_confidence,
+ Answer::Noul(n) => n.answer_confidence,
+ }
+ }
+
+ /// The action head's output.
+ pub fn action(&self) -> ActionInfo {
+ match self {
+ Answer::Choice(c) => c.action,
+ Answer::Score(s) => s.action,
+ Answer::Noul(n) => n.action,
+ }
+ }
+
+ /// This answer as a choice answer.
+ pub fn as_choice(&self) -> Option<&ChoiceAnswer> {
+ match self {
+ Answer::Choice(c) => Some(c),
+ _ => None,
+ }
+ }
+
+ /// This answer as a score answer.
+ pub fn as_score(&self) -> Option<&ScoreAnswer> {
+ match self {
+ Answer::Score(s) => Some(s),
+ _ => None,
+ }
+ }
+
+ /// This answer as a noul answer.
+ pub fn as_noul(&self) -> Option<&NoulAnswer> {
+ match self {
+ Answer::Noul(n) => Some(n),
+ _ => None,
+ }
+ }
+}
+
+/// The result of one `predict` call: one answer per question, plus token usage.
+#[derive(Debug, Clone)]
+pub struct LayaResult {
+ model: String,
+ usage: Usage,
+ order: Vec<String>,
+ by_id: HashMap<String, Answer>,
+ routing: Option<RouteDecision>,
+ shortlist: Option<IndexMap<String, ShortlistInfo>>,
+}
+
+impl LayaResult {
+ /// Build a result from the model identifier, question order, and per-id answers. `routing`
+ /// and `shortlist` start unset; see [`Self::with_routing`] and [`Self::with_shortlist`].
+ /// Ports Python's plain result dict, which `router.py`/`shortlist.py` add a `"routing"` /
+ /// `"shortlist"` key to after the fact — the same shape here as two optional fields, since
+ /// `LayaResult` is a typed struct rather than a dict.
+ pub fn new(
+ model: impl Into<String>,
+ order: Vec<String>,
+ by_id: HashMap<String, Answer>,
+ usage: Usage,
+ ) -> Self {
+ Self {
+ model: model.into(),
+ usage,
+ order,
+ by_id,
+ routing: None,
+ shortlist: None,
+ }
+ }
+
+ /// Attach a routing decision (`LayaRouter::predict` calls this on its own result before
+ /// returning it).
+ pub fn with_routing(mut self, decision: RouteDecision) -> Self {
+ self.routing = Some(decision);
+ self
+ }
+
+ /// Attach shortlist metadata, one entry per shortlisted choice question (`LayaShortlist::
+ /// predict` calls this on its own result before returning it).
+ pub fn with_shortlist(mut self, shortlist: IndexMap<String, ShortlistInfo>) -> Self {
+ self.shortlist = Some(shortlist);
+ self
+ }
+
+ /// The routing decision that selected this result's checkpoint, when this result came from
+ /// [`crate::laya_router::LayaRouter::predict`].
+ pub fn routing(&self) -> Option<&RouteDecision> {
+ self.routing.as_ref()
+ }
+
+ /// Per-question shortlist metadata, when this result came from
+ /// [`crate::laya_shortlist::LayaShortlist::predict`]. They compose: a shortlisted predict run
+ /// through a router carries both this and [`Self::routing`].
+ pub fn shortlist(&self) -> Option<&IndexMap<String, ShortlistInfo>> {
+ self.shortlist.as_ref()
+ }
+
+ /// The model identifier, matching Python's `"laya-rl-agent"`-style `model_name`.
+ pub fn model(&self) -> &str {
+ &self.model
+ }
+
+ /// Token accounting for the call.
+ pub fn usage(&self) -> Usage {
+ self.usage
+ }
+
+ /// The question ids, in the order they were asked.
+ pub fn ids(&self) -> &[String] {
+ &self.order
+ }
+
+ /// How many answers there are.
+ pub fn len(&self) -> usize {
+ self.order.len()
+ }
+
+ /// Whether there are no answers.
+ pub fn is_empty(&self) -> bool {
+ self.order.is_empty()
+ }
+
+ /// The answer to one question, if it was asked.
+ pub fn get(&self, id: &str) -> Option<&Answer> {
+ self.by_id.get(id)
+ }
+
+ /// Every `(id, answer)` pair, in the order the questions were asked.
+ pub fn iter(&self) -> impl Iterator<Item = (&str, &Answer)> {
+ self.order
+ .iter()
+ .map(move |id| (id.as_str(), &self.by_id[id]))
+ }
+}
+
+impl std::ops::Index<&str> for LayaResult {
+ type Output = Answer;
+
+ /// Panics if no question carried this id; use [`LayaResult::get`] for a fallible lookup.
+ fn index(&self, id: &str) -> &Answer {
+ self.by_id
+ .get(id)
+ .unwrap_or_else(|| panic!("no answer for question '{id}'"))
+ }
+}
diff --git a/laya-rust/src/calibration.rs b/laya-rust/src/calibration.rs
new file mode 100644
index 00000000..1d75c263
--- /dev/null
+++ b/laya-rust/src/calibration.rs
@@ -0,0 +1,163 @@
+//! Turns raw marker logits into calibrated probabilities and confidence. Ports `temp_bucket`,
+//! `clamp_temperature` and `confidence_from_probs` from `laya/common.py`, plus the readout
+//! arithmetic in `Agent.system_one`.
+//!
+//! Grouped as associated functions on a `Calibration` type rather than free functions, so the
+//! call sites read as `Calibration::temp_bucket(...)` etc.
+
+use crate::laya_config::LayaConfig;
+use crate::questions::QuestionType;
+use crate::sequence_builder::SequenceBuilder;
+
+/// Namespace for calibration math. See the module docs for the source this ports.
+pub struct Calibration;
+
+impl Calibration {
+ /// Lowest temperature that will be applied. A fitted temperature below 1 sharpens logits
+ /// instead of softening them; the shipped `choice:11+` bucket of 0.1006 multiplies them
+ /// roughly tenfold, publishing a 0.24 top probability as 0.99. A caller gating on confidence
+ /// would be told a coin flip is a certainty, so such a value is refused rather than applied.
+ pub const TEMP_MIN: f64 = 0.5;
+
+ /// Highest temperature that will be applied.
+ pub const TEMP_MAX: f64 = 5.0;
+
+ /// A usable temperature: confined to [`Self::TEMP_MIN`]..[`Self::TEMP_MAX`], with anything
+ /// that is not a finite number falling back to 1.0.
+ pub fn clamp_temperature(t: f64) -> f64 {
+ if !t.is_finite() {
+ 1.0
+ } else {
+ t.clamp(Self::TEMP_MIN, Self::TEMP_MAX)
+ }
+ }
+
+ /// The calibration bucket key for a question type and option count, e.g. `"choice:3-5"`.
+ pub fn temp_bucket(qtype: QuestionType, option_count: usize) -> String {
+ let size = if option_count <= 2 {
+ "2"
+ } else if option_count <= 5 {
+ "3-5"
+ } else if option_count <= 10 {
+ "6-10"
+ } else {
+ "11+"
+ };
+ format!("{}:{size}", SequenceBuilder::type_name(qtype))
+ }
+
+ /// The temperature to divide logits by for one question.
+ pub fn resolve_temperature(
+ config: &LayaConfig,
+ qtype: QuestionType,
+ option_count: usize,
+ ) -> f64 {
+ let bucket = Self::temp_bucket(qtype, option_count);
+ config
+ .temperature_by_options
+ .get(&bucket)
+ .copied()
+ .unwrap_or_else(|| config.temperature[qtype as usize])
+ }
+
+ /// Tempered softmax over the first `option_count` logits of one row.
+ ///
+ /// Only the real markers are read. Padding columns carry the `-1e4` the model's `masked_fill`
+ /// wrote, which would underflow to zero anyway, but slicing first keeps the normalisation
+ /// identical to Python's `logits[r, :k]`. Returns an empty vector for `option_count == 0`
+ /// rather than panicking — library code never panics on caller-controlled sizes.
+ pub fn probabilities(logits: &[f32], option_count: usize, temperature: f64) -> Vec<f64> {
+ let k = option_count.min(logits.len());
+ if k == 0 {
+ return Vec::new();
+ }
+
+ let mut p: Vec<f64> = logits[..k]
+ .iter()
+ .map(|&z| f64::from(z) / temperature)
+ .collect();
+ let max = p.iter().copied().fold(f64::NEG_INFINITY, f64::max);
+
+ let mut sum = 0.0;
+ for v in &mut p {
+ *v = (*v - max).exp();
+ sum += *v;
+ }
+ for v in &mut p {
+ *v /= sum;
+ }
+ p
+ }
+
+ /// Softmax over a whole row, used for the action head. No temperature scaling.
+ pub fn softmax(logits: &[f32]) -> Vec<f64> {
+ if logits.is_empty() {
+ return Vec::new();
+ }
+ let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let mut p: Vec<f64> = logits.iter().map(|&z| f64::from(z - max).exp()).collect();
+ let sum: f64 = p.iter().sum();
+ for v in &mut p {
+ *v /= sum;
+ }
+ p
+ }
+
+ /// Normalized Shannon entropy confidence, `1 - H(p) / log k`, clipped to `[0, 1]`. A
+ /// single-option question is fully decided by construction, so it returns 1.0.
+ pub fn confidence_from_probs(p: &[f64], option_count: usize) -> f64 {
+ if option_count < 2 {
+ return 1.0;
+ }
+ let k = option_count.min(p.len());
+ let mut entropy = 0.0;
+ for &pi in &p[..k] {
+ let clipped = pi.clamp(1e-12, 1.0);
+ entropy -= pi * clipped.ln();
+ }
+ (1.0 - entropy / (k as f64).ln()).clamp(0.0, 1.0)
+ }
+
+ /// Probability mass on the answer being reported: `max(p[:option_count])`, clipped to
+ /// `[0, 1]`. This is the quantity temperature scaling fits and every ECE figure in the Python
+ /// repository is computed on, unlike [`Self::confidence_from_probs`], which reports a
+ /// differently-scaled quantity with no such calibration guarantee. Ports `answer_confidence`
+ /// from `laya/common.py` (laya 0.3.21). Returns `1.0` when `option_count < 1`, matching
+ /// Python's `k < 1` guard.
+ pub fn answer_confidence(p: &[f64], option_count: usize) -> f64 {
+ if option_count < 1 {
+ return 1.0;
+ }
+ let k = option_count.min(p.len());
+ let max = p[..k].iter().copied().fold(f64::NEG_INFINITY, f64::max);
+ max.clamp(0.0, 1.0)
+ }
+
+ /// The expected level of a score answer: the probability-weighted mean of the level indices, a
+ /// fractional value rather than the most likely level.
+ pub fn expected_score(p: &[f64]) -> f64 {
+ p.iter().enumerate().map(|(i, &pi)| i as f64 * pi).sum()
+ }
+
+ /// The index of the largest probability, ties going to the lowest index as NumPy's `argmax`
+ /// does.
+ pub fn arg_max(p: &[f64]) -> usize {
+ let mut best = 0;
+ for (i, &v) in p.iter().enumerate().skip(1) {
+ if v > p[best] {
+ best = i;
+ }
+ }
+ best
+ }
+
+ /// Python's `round(x, 4)`: round-half-to-even at 4 decimal places. Both runtimes round half
+ /// to even, so small binary-vs-decimal tie discrepancies (at most ~5e-5) are well within the
+ /// parity tolerances used everywhere this is applied.
+ pub fn round4(value: f64) -> f64 {
+ if !value.is_finite() {
+ return value;
+ }
+ (value * 10_000.0).round_ties_even() / 10_000.0
+ }
+}
diff --git a/laya-rust/tests/calibration_tests.rs b/laya-rust/tests/calibration_tests.rs
new file mode 100644
index 00000000..e4eb50d7
--- /dev/null
+++ b/laya-rust/tests/calibration_tests.rs
@@ -0,0 +1,232 @@
+//! Tier 1: the calibration maths, which needs no artifacts.
+
+mod common;
+
+use common::golden_data::CheckpointGoldenData;
+use laya::calibration::Calibration;
+use laya::laya_config::LayaConfig;
+use laya::questions::QuestionType;
+
+#[test]
+fn temp_bucket_matches_python_boundaries() {
+ let cases = [
+ (QuestionType::Choice, 1, "choice:2"),
+ (QuestionType::Choice, 2, "choice:2"),
+ (QuestionType::Choice, 3, "choice:3-5"),
+ (QuestionType::Choice, 5, "choice:3-5"),
+ (QuestionType::Choice, 6, "choice:6-10"),
+ (QuestionType::Choice, 10, "choice:6-10"),
+ (QuestionType::Choice, 11, "choice:11+"),
+ (QuestionType::Score, 4, "score:3-5"),
+ (QuestionType::Noul, 2, "noul:2"),
+ ];
+ for (qtype, k, expected) in cases {
+ assert_eq!(
+ expected,
+ Calibration::temp_bucket(qtype, k),
+ "temp_bucket({qtype:?}, {k})"
+ );
+ }
+}
+
+#[test]
+fn clamp_temperature_matches_python() {
+ let cases = [
+ (1.0, 1.0),
+ (2.5, 2.5),
+ (0.1006, Calibration::TEMP_MIN), // the shipped choice:11+ value, refused as too sharp
+ (0.0, Calibration::TEMP_MIN),
+ (-3.0, Calibration::TEMP_MIN),
+ (99.0, Calibration::TEMP_MAX),
+ (f64::NAN, 1.0),
+ (f64::INFINITY, 1.0),
+ (f64::NEG_INFINITY, 1.0),
+ ];
+ for (input, expected) in cases {
+ assert_eq!(
+ expected,
+ Calibration::clamp_temperature(input),
+ "clamp_temperature({input})"
+ );
+ }
+}
+
+#[test]
+fn probabilities_read_only_the_real_markers() {
+ // The padded marker column carries the -1e4 the model's masked_fill wrote. A single-option
+ // question must still come out as a certainty rather than leaking the pad column in.
+ let p = Calibration::probabilities(&[-0.912_778, -10000.0], 1, 1.0);
+ assert_eq!(1, p.len());
+ assert!((p[0] - 1.0).abs() < 1e-12);
+}
+
+#[test]
+fn probabilities_sum_to_one_and_respect_temperature() {
+ let logits = [2.0f32, 1.0, 0.0];
+ let sharp = Calibration::probabilities(&logits, 3, 0.5);
+ let soft = Calibration::probabilities(&logits, 3, 5.0);
+
+ assert!((sharp.iter().sum::<f64>() - 1.0).abs() < 1e-12);
+ assert!((soft.iter().sum::<f64>() - 1.0).abs() < 1e-12);
+ // A lower temperature concentrates mass on the top option; a higher one flattens it.
+ assert!(sharp[0] > soft[0]);
+ assert!(soft[2] > sharp[2]);
+}
+
+#[test]
+fn probabilities_survive_large_logits() {
+ // act_logits in the goldens reach +/-1600; a naive exp() would overflow to NaN.
+ let p = Calibration::softmax(&[1366.2135, -1624.665]);
+ assert!((p[0] - 1.0).abs() < 1e-12);
+ assert!((p[1] - 0.0).abs() < 1e-12);
+}
+
+#[test]
+fn confidence_is_one_for_a_single_option() {
+ assert_eq!(1.0, Calibration::confidence_from_probs(&[1.0], 1));
+}
+
+#[test]
+fn confidence_is_zero_for_a_uniform_distribution() {
+ assert!((Calibration::confidence_from_probs(&[0.5, 0.5], 2) - 0.0).abs() < 1e-12);
+ assert!((Calibration::confidence_from_probs(&[0.25, 0.25, 0.25, 0.25], 4) - 0.0).abs() < 1e-12);
+}
+
+#[test]
+fn confidence_is_one_for_a_certainty() {
+ assert!((Calibration::confidence_from_probs(&[1.0, 0.0], 2) - 1.0).abs() < 1e-12);
+}
+
+#[test]
+fn confidence_matches_the_recorded_quickstart_values() {
+ // From golden/multilingual/case_quickstart.json: urgency's four-level distribution and confidence.
+ let urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ let confidence = Calibration::round4(Calibration::confidence_from_probs(&urgency, 4));
+ assert!(
+ (confidence - 0.1976).abs() < 1e-3,
+ "confidence = {confidence}"
+ );
+}
+
+#[test]
+fn confidence_ignores_padding_beyond_the_option_count() {
+ let padded = [0.5, 0.5, 0.0, 0.0];
+ assert_eq!(
+ Calibration::confidence_from_probs(&[0.5, 0.5], 2),
+ Calibration::confidence_from_probs(&padded, 2)
+ );
+}
+
+#[test]
+fn answer_confidence_is_the_top_probability_not_the_entropy_gap() {
+ // Ports laya/common.py's `answer_confidence` (laya 0.3.21): unlike `confidence_from_probs`
+ // (normalized entropy), it is just `max(p[:k])`, clipped to [0, 1].
+ assert_eq!(1.0, Calibration::answer_confidence(&[1.0], 1));
+ assert_eq!(0.5, Calibration::answer_confidence(&[0.5, 0.5], 2));
+ assert_eq!(0.7, Calibration::answer_confidence(&[0.7, 0.2, 0.1], 3));
+}
+
+#[test]
+fn answer_confidence_ignores_padding_beyond_the_option_count() {
+ let padded = [0.5, 0.5, 0.9, 0.9];
+ assert_eq!(
+ Calibration::answer_confidence(&[0.5, 0.5], 2),
+ Calibration::answer_confidence(&padded, 2)
+ );
+}
+
+#[test]
+fn answer_confidence_is_one_when_option_count_is_less_than_one() {
+ // Mirrors Python's `if k < 1: return 1.0` guard.
+ assert_eq!(1.0, Calibration::answer_confidence(&[0.5, 0.5], 0));
+}
+
+#[test]
+fn answer_confidence_matches_the_recorded_quickstart_values() {
+ // Same distribution as confidence_matches_the_recorded_quickstart_values, but the top
+ // probability rather than the normalized-entropy confidence.
+ let urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ let answer_confidence = Calibration::round4(Calibration::answer_confidence(&urgency, 4));
+ assert!(
+ (answer_confidence - 0.465).abs() < 1e-3,
+ "answer_confidence = {answer_confidence}"
+ );
+}
+
+#[test]
+fn expected_score_is_a_probability_weighted_mean() {
+ let urgency = [0.0179, 0.465, 0.3208, 0.1964];
+ let expected = Calibration::round4(Calibration::expected_score(&urgency));
+ assert!(
+ (expected - 1.6956).abs() < 1e-3,
+ "expected_score = {expected}"
+ );
+ assert!((Calibration::expected_score(&[1.0, 0.0, 0.0]) - 0.0).abs() < 1e-12);
+ assert!((Calibration::expected_score(&[0.0, 0.0, 1.0]) - 2.0).abs() < 1e-12);
+}
+
+#[test]
+fn arg_max_breaks_ties_toward_the_lowest_index_as_numpy_does() {
+ assert_eq!(0, Calibration::arg_max(&[0.5, 0.5, 0.0]));
+}
+
+#[test]
+fn resolve_temperature_prefers_the_bucket_then_the_type_default() {
+ let config = LayaConfig::parse(
+ r#"{"max_len": 1024, "head_max_len": 256,
+ "temperature": [1.5, 2.0, 2.5],
+ "temperature_by_options": {"choice:3-5": 3.0, "noul:2": 0.2}}"#,
+ )
+ .expect("valid config");
+
+ assert_eq!(
+ 3.0,
+ Calibration::resolve_temperature(&config, QuestionType::Choice, 4)
+ );
+ assert_eq!(
+ 1.5,
+ Calibration::resolve_temperature(&config, QuestionType::Choice, 8)
+ );
+ assert_eq!(
+ 2.0,
+ Calibration::resolve_temperature(&config, QuestionType::Score, 4)
+ );
+ // 0.2 is below TEMP_MIN, so the bucket is clamped rather than applied as recorded.
+ assert_eq!(
+ Calibration::TEMP_MIN,
+ Calibration::resolve_temperature(&config, QuestionType::Noul, 2)
+ );
+}
+
+#[test]
+fn config_falls_back_to_pythons_in_code_defaults() {
+ let config = LayaConfig::parse("{}").expect("empty config parses");
+ assert_eq!(512, config.max_len);
+ assert_eq!(192, config.head_max_len);
+ assert_eq!(vec![1.0, 1.0, 1.0], config.temperature);
+}
+
+#[test]
+fn shipped_config_parses_as_recorded() {
+ let data = CheckpointGoldenData::for_checkpoint("multilingual");
+ if !data.available() {
+ eprintln!("skip: no golden data for 'multilingual'");
+ return;
+ }
+ let meta = data.meta();
+ let config = LayaConfig::parse(&meta["config"].to_string()).expect("shipped config parses");
+
+ assert_eq!(meta["max_len"].as_u64().unwrap() as usize, config.max_len);
+ assert_eq!(
+ meta["head_max_len"].as_u64().unwrap() as usize,
+ config.head_max_len
+ );
+ let expected_temperature: Vec<f64> = meta["temperature"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|v| v.as_f64().unwrap())
+ .collect();
+ assert_eq!(expected_temperature, config.temperature);
+ assert!(config.temperature_by_options.is_empty());
+}