Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(calibration): handle invalid entries when reporting temperatures (#142)
Alphin Tomba1863e98e7d
diff --git a/README.md b/README.mdindex 08f6a88c..d2963eee 100644--- a/README.md+++ b/README.md@@ -392,6 +392,13 @@ option count) on held-out data moves mean ECE **0.466 -> 0.081** (`laya`) and**0.314 -> 0.106** (`laya-multilingual`). `laya-multilingual` ships with no fittedtemperatures at all, so fit them before relying on its probabilities.+At checkpoint load, numeric temperature entries are clamped to `[0.5, 5.0]`; invalid or+non-finite entries use the neutral fallback `1.0`. A runtime warning reports the affected+entries and applied values. Bucket-specific temperatures still take precedence over per-type+values, including when a bucket uses the fallback. Raw values remain available in+`agent.temperature_raw` and `agent.temperature_by_options_raw`. A fallback prevents a loading+failure; it does not establish calibrated confidence.+### Honest limits* **The base checkpoints are near chance on typed-decisions zero-shot** -- 0.362 and 0.352diff --git a/laya/agent.py b/laya/agent.pyindex 5017904b..b938758f 100644--- a/laya/agent.py+++ b/laya/agent.py@@ -215,14 +215,22 @@ class Agent:self.temperature = [clamp_temperature(t) for t in self.temperature_raw]self.temperature_by_options = {k: clamp_temperature(v)for k, v in self.temperature_by_options_raw.items()}- rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()- if clamp_temperature(v) != float(v)]- rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)- if clamp_temperature(t) != float(t)]+ entries = [(k, v, self.temperature_by_options[k]) for k, v in self.temperature_by_options_raw.items()]+ entries += [("temperature[%d]" % i, t, self.temperature[i]) for i, t in enumerate(self.temperature_raw)]+ rejected = []+ for name, raw, applied in entries:+ try:+ if float(raw) == applied:+ continue+ except (TypeError, ValueError):+ # Invalid entries already have a neutral fallback; diagnostics must not+ # repeat the failed conversion or prevent the checkpoint from loading.+ pass+ rejected.append("%s=%r -> %g" % (name, raw, applied))if rejected:warnings.warn(- "laya: this checkpoint ships temperatures outside [%g, %g] which would distort "- "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."+ "laya: this checkpoint ships invalid temperatures or values outside [%g, %g]; "+ "using %s. Treat confidence from the affected entries as uncalibrated."% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),RuntimeWarning, stacklevel=2)self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))diff --git a/tests/test_temperature_loading.py b/tests/test_temperature_loading.pynew file mode 100644index 00000000..d69aa180--- /dev/null+++ b/tests/test_temperature_loading.py@@ -0,0 +1,156 @@+"""Temperature loading regressions using a tiny local checkpoint; no training or downloads.++Run: python tests/test_temperature_loading.py+"""+import json+import os+from pathlib import Path+import sys+import tempfile+import unittest+from unittest.mock import patch+import warnings++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("HF_HUB_OFFLINE", "1")+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import torch # noqa: E402+from safetensors.torch import save_file # noqa: E402+from tokenizers import Tokenizer # noqa: E402+from tokenizers.models import WordLevel # noqa: E402+from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402++from laya import load # noqa: E402+from laya.common import DecisionModel # noqa: E402+++class TemperatureLoadingTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.tmp = tempfile.TemporaryDirectory()+ cls.addClassCleanup(cls.tmp.cleanup)+ cls.repo = Path(cls.tmp.name)+ config = BertConfig(vocab_size=6, hidden_size=16, num_hidden_layers=1,+ num_attention_heads=1, intermediate_size=32)+ config.save_pretrained(cls.repo / "encoder")+ tokenizer = PreTrainedTokenizerFast(+ tokenizer_object=Tokenizer(WordLevel(+ {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "hello": 5},+ unk_token="[UNK]")),+ pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",+ sep_token="[SEP]", mask_token="[MASK]",+ )+ tokenizer.save_pretrained(cls.repo / "tokenizer")+ model = DecisionModel(BertModel(config), head_layers=0)+ save_file(model.state_dict(), cls.repo / "model.safetensors")++ def load_config(self, **temperatures):+ cfg = {"encoder": "unused/offline", "head_layers": 0, "act_costs": {"act": 0},+ "max_len": 64, "head_max_len": 32, **temperatures}+ path = self.repo / "rl_agent_config.json"+ path.write_text(json.dumps(cfg))+ original = path.read_bytes()+ with warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):+ agent = load(str(self.repo), device="cpu")+ self.assertEqual(agent.device.type, "cpu")+ self.assertEqual(path.read_bytes(), original)+ # JSON comparison also handles NaN, which is unequal to itself.+ self.assertEqual(json.dumps(agent.cfg), json.dumps(cfg))+ self.assertEqual(json.dumps(agent.temperature_raw), json.dumps(cfg.get("temperature", [1.0] * 3)))+ self.assertEqual(json.dumps(agent.temperature_by_options_raw), json.dumps(cfg.get("temperature_by_options", {})))+ messages = [w for w in caught if str(w.message).startswith("laya:")]+ self.assertTrue(all(issubclass(w.category, RuntimeWarning) for w in messages))+ return agent, [str(w.message) for w in messages]++ def assert_predictions(self, agent, choice, score, noul):+ questions = {+ "choice": {"type": "choice", "instructions": "Pick one", "criteria": ["a", "b"]},+ "score": {"type": "score", "instructions": "Rate", "criteria": ["low", "mid", "high"]},+ "noul": {"type": "noul", "instructions": "True?"},+ }+ # Loading is real; fixed logits make the expected scaling independent of random weights.+ logits = torch.tensor([[0.0, 1.0, -1e4], [0.0, 1.0, 2.0], [0.0, 1.0, -1e4]])+ with patch.object(agent.model, "forward", return_value=(logits, torch.zeros(3, 2))):+ answers = agent.predict("hello", questions)["answers"]+ for name, temperature, k in (("choice", choice, 2), ("score", score, 3), ("noul", noul, 2)):+ p = torch.softmax(torch.arange(k, dtype=torch.float32) / temperature, -1)+ if name == "noul":+ self.assertAlmostEqual(answers[name]["noul"], p[1].item(), delta=0.0001)+ else:+ actual = list(answers[name]["probabilities"].values())+ self.assertEqual(len(actual), k)+ for got, want in zip(actual, p.tolist()):+ self.assertAlmostEqual(got, want, delta=0.0001)++ def test_invalid_type_entries_use_neutral_fallback_and_warn(self):+ for value in (None, "invalid", "", [], {}):+ with self.subTest(value=value):+ agent, messages = self.load_config(temperature=[value, 2.0, 3.0])+ self.assertEqual(agent.temperature, [1.0, 2.0, 3.0])+ self.assertEqual(len(messages), 1)+ self.assertIn("temperature[0]", messages[0])+ self.assertIn(repr(value), messages[0])+ self.assertIn("uncalibrated", messages[0])+ self.assert_predictions(agent, choice=1.0, score=2.0, noul=3.0)++ def test_invalid_bucket_entries_use_neutral_fallback_and_keep_precedence(self):+ for value in (None, "invalid", "", [], {}):+ with self.subTest(value=value):+ agent, messages = self.load_config(temperature=[2.0, 3.0, 4.0],+ temperature_by_options={"choice:2": value})+ self.assertEqual(agent.temperature_by_options, {"choice:2": 1.0})+ self.assertEqual(len(messages), 1)+ self.assertIn("choice:2", messages[0])+ self.assertIn(repr(value), messages[0])+ self.assert_predictions(agent, choice=1.0, score=3.0, noul=4.0)++ def test_nonfinite_values_keep_neutral_fallback_and_warn(self):+ for value in (float("nan"), float("inf"), float("-inf"), "NaN", "Infinity", "-Infinity"):+ with self.subTest(value=value):+ agent, messages = self.load_config(temperature=[value, 2.0, 3.0],+ temperature_by_options={"noul:2": value})+ self.assertEqual(agent.temperature, [1.0, 2.0, 3.0])+ self.assertEqual(agent.temperature_by_options, {"noul:2": 1.0})+ self.assertEqual(len(messages), 1)+ self.assertIn("temperature[0]", messages[0])+ self.assertIn("noul:2", messages[0])+ self.assert_predictions(agent, choice=1.0, score=2.0, noul=1.0)++ def test_out_of_range_values_keep_existing_clamps(self):+ agent, messages = self.load_config(temperature=[0.1, 10, -1],+ temperature_by_options={"choice:2": 9, "score:3-5": 0})+ self.assertEqual(agent.temperature, [0.5, 5.0, 0.5])+ self.assertEqual(agent.temperature_by_options, {"choice:2": 5.0, "score:3-5": 0.5})+ self.assertEqual(len(messages), 1)+ for entry in ("temperature[0]", "temperature[1]", "temperature[2]", "choice:2", "score:3-5"):+ self.assertIn(entry, messages[0])+ self.assert_predictions(agent, choice=5.0, score=0.5, noul=0.5)++ def test_valid_numbers_and_numeric_strings_do_not_warn(self):+ agent, messages = self.load_config(temperature=[0.5, "2", 5],+ temperature_by_options={"choice:2": "1.5", "score:3-5": 2.5, "noul:2": "5.0"})+ self.assertEqual(agent.temperature, [0.5, 2.0, 5.0])+ self.assertEqual(agent.temperature_by_options, {"choice:2": 1.5, "score:3-5": 2.5, "noul:2": 5.0})+ self.assertEqual(messages, [])+ self.assert_predictions(agent, choice=1.5, score=2.5, noul=5.0)++ def test_missing_bucket_uses_corresponding_type(self):+ agent, messages = self.load_config(temperature=[2.0, 3.0, 4.0], temperature_by_options={"choice:2": 1.5})+ self.assertEqual(messages, [])+ self.assert_predictions(agent, choice=1.5, score=3.0, noul=4.0)++ def test_missing_temperature_fields_keep_defaults(self):+ agent, messages = self.load_config()+ self.assertEqual(agent.temperature, [1.0, 1.0, 1.0])+ self.assertEqual(agent.temperature_by_options, {})+ self.assertEqual(messages, [])+ self.assert_predictions(agent, choice=1.0, score=1.0, noul=1.0)+++if __name__ == "__main__":+ torch.set_num_threads(1)+ unittest.main()