Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(rust): validate action probability parity across CPU kernels

kevin-gatimu6476b7ef54d9
diff --git a/laya-rust/README.md b/laya-rust/README.md
index ec9a0b52..4a997bf1 100644
--- a/laya-rust/README.md
+++ b/laya-rust/README.md
@@ -557,6 +557,12 @@ with no path filter. It regenerates fixtures from the Python `laya/` at that com
Goldens are never cached. A `skip:` diagnostic or ignored test makes the lane red, even when
Cargo counts a missing-model early return as a pass. Job-level `continue-on-error` keeps Python
improvements unblocked; Rust drift needs a follow-up port. This workflow never publishes crates.
+The regeneration self-check compares option logits within `1e-3` and every **unrounded action
+softmax probability** within `1e-4`, matching the signal Python and Rust predict consume.
+Action classes are not padded option slots and none are excluded. Raw action logits remain
+recorded (and their max difference reported) for diagnostics, but large saturated logits can
+vary across CPU kernels without changing action probabilities. Nonfinite or mismatched-shaped
+outputs are rejected. The recorded-graph Rust test uses the same action probability bound.
Engine results expose Python 0.4.0's state-budget diagnostics via `LayaResult::state_usage()`:
original state tokens, worst-case dropped tokens, whether truncation occurred, and the affected
diff --git a/laya-rust/tests/golden/english/meta.json b/laya-rust/tests/golden/english/meta.json
index 80cabdcd..24260c96 100644
--- a/laya-rust/tests/golden/english/meta.json
+++ b/laya-rust/tests/golden/english/meta.json
@@ -70,12 +70,14 @@
},
"onnx_self_check": {
"single_noul": {
- "logits_maxdiff": 9.953975677490234e-06,
- "act_maxdiff": 0.0009765625
+ "logits_maxdiff": 2.3871660232543945e-05,
+ "act_maxdiff": 0.001953125,
+ "act_probs_maxdiff": 0.0
},
"triage_4q": {
- "logits_maxdiff": 1.946091651916504e-05,
- "act_maxdiff": 0.00146484375
+ "logits_maxdiff": 1.823902130126953e-05,
+ "act_maxdiff": 0.00146484375,
+ "act_probs_maxdiff": 0.0
}
}
}
diff --git a/laya-rust/tests/golden/multilingual/meta.json b/laya-rust/tests/golden/multilingual/meta.json
index 3cdc37c3..e57fdd98 100644
--- a/laya-rust/tests/golden/multilingual/meta.json
+++ b/laya-rust/tests/golden/multilingual/meta.json
@@ -57,11 +57,13 @@
"onnx_self_check": {
"single_noul": {
"logits_maxdiff": 1.5795230865478516e-06,
- "act_maxdiff": 0.0001220703125
+ "act_maxdiff": 0.0001220703125,
+ "act_probs_maxdiff": 0.0
},
"triage_4q": {
"logits_maxdiff": 1.3232231140136719e-05,
- "act_maxdiff": 0.0009765625
+ "act_maxdiff": 0.0009765625,
+ "act_probs_maxdiff": 0.0
}
}
}
diff --git a/laya-rust/tests/golden/typed-decisions/meta.json b/laya-rust/tests/golden/typed-decisions/meta.json
index f7b16d25..fbb10bf1 100644
--- a/laya-rust/tests/golden/typed-decisions/meta.json
+++ b/laya-rust/tests/golden/typed-decisions/meta.json
@@ -74,11 +74,13 @@
"onnx_self_check": {
"single_noul": {
"logits_maxdiff": 4.470348358154297e-07,
- "act_maxdiff": 0.00146484375
+ "act_maxdiff": 0.00146484375,
+ "act_probs_maxdiff": 0.0
},
"triage_4q": {
"logits_maxdiff": 2.6226043701171875e-06,
- "act_maxdiff": 0.00146484375
+ "act_maxdiff": 0.00146484375,
+ "act_probs_maxdiff": 0.0
}
}
}
diff --git a/laya-rust/tests/shape_sweep_tests.rs b/laya-rust/tests/shape_sweep_tests.rs
index 7057edf6..5b9fbf47 100644
--- a/laya-rust/tests/shape_sweep_tests.rs
+++ b/laya-rust/tests/shape_sweep_tests.rs
@@ -1,4 +1,4 @@
-//! Tier 2: the exported graph's dynamic axes and its raw outputs, driven directly through
+//! Tier 2: the exported graph's dynamic axes, option logits and action probabilities, driven through
//! [`laya::LayaEngine::run`].
//!
//! The export was traced at one shape, so every axis being genuinely dynamic is a property of the
@@ -14,6 +14,7 @@ mod common;
use common::engines;
use common::golden_data::{self, CheckpointGoldenData};
+use laya::calibration::Calibration;
use laya::collator::{CollatedBatch, Collator, SequenceItem};
use laya::laya_config::LayaConfig;
use laya::laya_options::LayaCheckpoint;
@@ -163,6 +164,10 @@ fn assert_close_slices(expected: &[f32], actual: &[f32], tolerance: f64, what: &
let mut worst = 0.0f64;
let mut at = 0usize;
for (i, (&e, &a)) in expected.iter().zip(actual.iter()).enumerate() {
+ assert!(
+ e.is_finite() && a.is_finite(),
+ "{what}: nonfinite value at index {i}"
+ );
let diff = (f64::from(e) - f64::from(a)).abs();
if diff > worst {
worst = diff;
@@ -178,6 +183,53 @@ fn assert_close_slices(expected: &[f32], actual: &[f32], tolerance: f64, what: &
);
}
+fn assert_act_probabilities(expected: &[f32], actual: &[f32], classes: usize, what: &str) {
+ assert!(classes > 0, "{what}: no action classes");
+ assert_eq!(expected.len(), actual.len(), "{what}: length mismatch");
+ assert_eq!(expected.len() % classes, 0, "{what}: incomplete action row");
+ assert!(
+ expected.iter().chain(actual).all(|v| v.is_finite()),
+ "{what}: nonfinite logits"
+ );
+ for (row, (e, a)) in expected
+ .chunks_exact(classes)
+ .zip(actual.chunks_exact(classes))
+ .enumerate()
+ {
+ let e: Vec<f32> = Calibration::softmax(e)
+ .into_iter()
+ .map(|v| v as f32)
+ .collect();
+ let a: Vec<f32> = Calibration::softmax(a)
+ .into_iter()
+ .map(|v| v as f32)
+ .collect();
+ assert_close_slices(&e, &a, 1e-4, &format!("{what} row {row}"));
+ }
+}
+
+#[test]
+fn saturated_act_logit_drift_preserves_probabilities() {
+ assert_act_probabilities(
+ &[4807.720, -3937.599],
+ &[4807.878, -3937.442],
+ 2,
+ "saturated",
+ );
+}
+
+#[test]
+#[should_panic(expected = "worst difference")]
+fn small_act_logit_drift_with_real_signal_is_rejected() {
+ assert_act_probabilities(&[0.0, 0.0], &[0.005, 0.0], 2, "signal");
+}
+
+#[test]
+#[should_panic(expected = "nonfinite logits")]
+fn nonfinite_act_logits_are_rejected() {
+ assert_act_probabilities(&[0.0, 0.0], &[f32::NAN, 0.0], 2, "invalid");
+}
+
// ── multilingual-only shape tests ────────────────────────────────────────────
#[test]
@@ -250,13 +302,15 @@ fn reproduces_the_recorded_graph_outputs() {
&format!("{case}: logits"),
);
- // act_logits reach +/-1600, where float32 spacing alone is ~1e-4; 0.05 is ~3e-5 relative.
+ // Match the unrounded action signal predict consumes, not CPU-kernel drift
+ // in large saturated logits. Keep the raw logits in the goldens for diagnostics.
let expected_act = golden_data::matrix(&root["act_logits"], &mut rows, &mut cols);
- assert_close_slices(
+ assert_eq!(batch.count, rows, "{case}: action row count");
+ assert_act_probabilities(
&expected_act,
&output.act_logits,
- 0.05,
- &format!("{case}: act_logits"),
+ cols,
+ &format!("{case}: act_probs"),
);
}
}
diff --git a/laya-rust/tools/dump_golden.py b/laya-rust/tools/dump_golden.py
index 7a531e1a..6c410e46 100644
--- a/laya-rust/tools/dump_golden.py
+++ b/laya-rust/tools/dump_golden.py
@@ -8,8 +8,9 @@ unmodified, so every number written here is what Python actually answers -- not
paraphrase of it. It also means no 1.3 GB safetensors checkpoint is needed: the exported
ONNX graph supplies the forward pass.
-Before writing anything the script checks its ONNX session against the PyTorch reference
-vectors bundled in `fixtures.npz`, so a bad session can never silently poison the goldens.
+Before writing anything the script checks option logits and unrounded action probabilities
+against the PyTorch reference vectors bundled in `fixtures.npz`. Raw action logits are
+retained in the goldens and the self-check report for diagnostics.
Usage:
# Explicit paths (original interface, still works):
@@ -250,17 +251,32 @@ def self_check(agent: Agent, onnx_dir: str) -> dict:
if not all("%s/%s" % (g, n) in z.files for n in need):
continue
logits, act = agent.model(*[torch.from_numpy(z["%s/%s" % (g, n)]) for n in need])
- d_logits = float(np.abs(logits.numpy() - z["%s/ref_logits" % g]).max())
- d_act = float(np.abs(act.numpy() - z["%s/ref_act" % g]).max())
- report[g] = {"logits_maxdiff": d_logits, "act_maxdiff": d_act}
- # fp32 ONNX vs the fp32 torch reference: 1e-3 is loose enough for kernel
- # reassociation and tight enough that a wrong graph cannot pass.
- if d_logits > 1e-3 or d_act > 1e-2:
+ ref_logits = z["%s/ref_logits" % g]
+ ref_act = z["%s/ref_act" % g]
+ for actual, reference in [(logits.numpy(), ref_logits), (act.numpy(), ref_act)]:
+ if actual.shape != reference.shape or not (
+ np.isfinite(actual).all() and np.isfinite(reference).all()
+ ):
+ raise SystemExit("invalid ONNX/reference outputs for %r -- refusing to emit goldens" % g)
+ d_logits = float(np.abs(logits.numpy() - ref_logits).max())
+ d_act = float(np.abs(act.numpy() - ref_act).max())
+ # Agent._forward consumes softmax(act), not raw act logits. Check every
+ # action probability before rounding: saturated logits in the thousands
+ # can drift across CPU kernels without changing that signal at all.
+ # Do not mask action columns: they are classes, not padded option slots.
+ act_probs = torch.softmax(act.float(), -1).numpy()
+ ref_probs = torch.softmax(torch.from_numpy(ref_act).float(), -1).numpy()
+ d_act_probs = float(np.abs(act_probs - ref_probs).max())
+ report[g] = {"logits_maxdiff": d_logits, "act_maxdiff": d_act,
+ "act_probs_maxdiff": d_act_probs}
+ if d_logits > 1e-3 or d_act_probs > 1e-4:
raise SystemExit(
"ONNX session does not match the PyTorch reference for %r "
- "(logits %.3e, act %.3e) -- refusing to emit goldens" % (g, d_logits, d_act)
+ "(logits %.3e, act %.3e, act_probs %.3e) -- refusing to emit goldens"
+ % (g, d_logits, d_act, d_act_probs)
)
- print(" self-check %-14s logits %.3e act %.3e OK" % (g, d_logits, d_act))
+ print(" self-check %-14s logits %.3e act %.3e act_probs %.3e OK"
+ % (g, d_logits, d_act, d_act_probs))
if not report:
raise SystemExit("fixtures.npz contained no usable reference groups")
return report
diff --git a/laya-rust/tools/test_ci_tools.py b/laya-rust/tools/test_ci_tools.py
index edcd711e..487e4f84 100644
--- a/laya-rust/tools/test_ci_tools.py
+++ b/laya-rust/tools/test_ci_tools.py
@@ -147,5 +147,65 @@ class FloatProbeTests(unittest.TestCase):
)
+class SelfCheckTests(unittest.TestCase):
+ def check(self, actual_act, reference_act, actual_logits=None):
+ import numpy as np
+ import torch
+ import dump_golden as dump
+
+ arrays = {f"single_noul/{n}": np.zeros((1, 2), dtype=np.int64)
+ for n in ["input_ids", "attention_mask", "marker_pos", "marker_mask", "qtype"]}
+ arrays["single_noul/ref_logits"] = np.array([[0.1, 0.2]], dtype=np.float32)
+ arrays["single_noul/ref_act"] = np.array([reference_act], dtype=np.float32)
+
+ class Fixture:
+ files = list(arrays)
+
+ def __getitem__(self, key):
+ return arrays[key]
+
+ logits = arrays["single_noul/ref_logits"] if actual_logits is None else np.array(
+ [actual_logits], dtype=np.float32
+ )
+ agent = types.SimpleNamespace(model=lambda *args: (
+ torch.from_numpy(logits), torch.tensor([actual_act], dtype=torch.float32)
+ ))
+ with patch.object(dump.os.path, "exists", return_value=True), patch.object(
+ dump.np, "load", return_value=Fixture()
+ ), contextlib.redirect_stdout(io.StringIO()):
+ return dump.self_check(agent, "artifacts")
+
+ def test_saturated_act_logits_may_drift_without_changing_signal(self):
+ report = self.check([4807.878, -3937.442], [4807.720, -3937.599])
+ self.assertGreater(report["single_noul"]["act_maxdiff"], 0.15)
+ self.assertEqual(0.0, report["single_noul"]["act_probs_maxdiff"])
+
+ def test_real_probability_drift_is_rejected_even_below_old_logit_tolerance(self):
+ with self.assertRaisesRegex(SystemExit, "act_probs.*refusing"):
+ self.check([0.005, 0.0], [0.0, 0.0])
+
+ def test_saturated_action_flip_is_rejected(self):
+ with self.assertRaises(SystemExit):
+ self.check([-3937.599, 4807.720], [4807.720, -3937.599])
+
+ def test_small_unsaturated_probability_drift_is_accepted(self):
+ report = self.check([0.0002, 0.0], [0.0, 0.0])
+ self.assertLess(report["single_noul"]["act_probs_maxdiff"], 1e-4)
+
+ def test_option_logit_drift_remains_rejected(self):
+ with self.assertRaises(SystemExit):
+ self.check([0.0, 0.0], [0.0, 0.0], [0.102, 0.2])
+
+ def test_nonfinite_or_wrong_shape_outputs_fail_closed(self):
+ for actual, reference in [
+ ([float("nan"), 0.0], [0.0, 0.0]),
+ ([float("inf"), 0.0], [0.0, 0.0]),
+ ([0.0, 0.0], [float("nan"), 0.0]),
+ ([0.0], [0.0, 0.0]),
+ ]:
+ with self.subTest(actual=actual, reference=reference), self.assertRaises(SystemExit):
+ self.check(actual, reference)
+
+
if __name__ == "__main__":
unittest.main()