Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(calibration): handle invalid entries when reporting temperatures (#142)

Alphin Tomba1863e98e7d
diff --git a/README.md b/README.md
index 08f6a88c..d2963eee 100644
--- a/README.md
+++ b/README.md
@@ -392,6 +392,13 @@ option count) on held-out data moves mean ECE **0.466 -> 0.081** (`laya`) and
**0.314 -> 0.106** (`laya-multilingual`). `laya-multilingual` ships with no fitted
temperatures at all, so fit them before relying on its probabilities.
+At checkpoint load, numeric temperature entries are clamped to `[0.5, 5.0]`; invalid or
+non-finite entries use the neutral fallback `1.0`. A runtime warning reports the affected
+entries and applied values. Bucket-specific temperatures still take precedence over per-type
+values, including when a bucket uses the fallback. Raw values remain available in
+`agent.temperature_raw` and `agent.temperature_by_options_raw`. A fallback prevents a loading
+failure; it does not establish calibrated confidence.
+
### Honest limits
* **The base checkpoints are near chance on typed-decisions zero-shot** -- 0.362 and 0.352
diff --git a/laya/agent.py b/laya/agent.py
index 5017904b..b938758f 100644
--- a/laya/agent.py
+++ b/laya/agent.py
@@ -215,14 +215,22 @@ class Agent:
self.temperature = [clamp_temperature(t) for t in self.temperature_raw]
self.temperature_by_options = {k: clamp_temperature(v)
for k, v in self.temperature_by_options_raw.items()}
- rejected = ["%s=%.4g" % (k, float(v)) for k, v in self.temperature_by_options_raw.items()
- if clamp_temperature(v) != float(v)]
- rejected += ["temperature[%d]=%.4g" % (i, float(t)) for i, t in enumerate(self.temperature_raw)
- if clamp_temperature(t) != float(t)]
+ entries = [(k, v, self.temperature_by_options[k]) for k, v in self.temperature_by_options_raw.items()]
+ entries += [("temperature[%d]" % i, t, self.temperature[i]) for i, t in enumerate(self.temperature_raw)]
+ rejected = []
+ for name, raw, applied in entries:
+ try:
+ if float(raw) == applied:
+ continue
+ except (TypeError, ValueError):
+ # Invalid entries already have a neutral fallback; diagnostics must not
+ # repeat the failed conversion or prevent the checkpoint from loading.
+ pass
+ rejected.append("%s=%r -> %g" % (name, raw, applied))
if rejected:
warnings.warn(
- "laya: this checkpoint ships temperatures outside [%g, %g] which would distort "
- "confidence; clamping %s. Treat confidence from the affected buckets as uncalibrated."
+ "laya: this checkpoint ships invalid temperatures or values outside [%g, %g]; "
+ "using %s. Treat confidence from the affected entries as uncalibrated."
% (TEMP_MIN, TEMP_MAX, ", ".join(rejected)),
RuntimeWarning, stacklevel=2)
self.dtype = amp_dtype(self.cfg.get("amp_dtype", "fp16"))
diff --git a/tests/test_temperature_loading.py b/tests/test_temperature_loading.py
new file mode 100644
index 00000000..d69aa180
--- /dev/null
+++ b/tests/test_temperature_loading.py
@@ -0,0 +1,156 @@
+"""Temperature loading regressions using a tiny local checkpoint; no training or downloads.
+
+Run: python tests/test_temperature_loading.py
+"""
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+import warnings
+
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("USE_TORCH", "1")
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import torch # noqa: E402
+from safetensors.torch import save_file # noqa: E402
+from tokenizers import Tokenizer # noqa: E402
+from tokenizers.models import WordLevel # noqa: E402
+from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
+
+from laya import load # noqa: E402
+from laya.common import DecisionModel # noqa: E402
+
+
+class TemperatureLoadingTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.addClassCleanup(cls.tmp.cleanup)
+ cls.repo = Path(cls.tmp.name)
+ config = BertConfig(vocab_size=6, hidden_size=16, num_hidden_layers=1,
+ num_attention_heads=1, intermediate_size=32)
+ config.save_pretrained(cls.repo / "encoder")
+ tokenizer = PreTrainedTokenizerFast(
+ tokenizer_object=Tokenizer(WordLevel(
+ {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4, "hello": 5},
+ unk_token="[UNK]")),
+ pad_token="[PAD]", unk_token="[UNK]", cls_token="[CLS]",
+ sep_token="[SEP]", mask_token="[MASK]",
+ )
+ tokenizer.save_pretrained(cls.repo / "tokenizer")
+ model = DecisionModel(BertModel(config), head_layers=0)
+ save_file(model.state_dict(), cls.repo / "model.safetensors")
+
+ def load_config(self, **temperatures):
+ cfg = {"encoder": "unused/offline", "head_layers": 0, "act_costs": {"act": 0},
+ "max_len": 64, "head_max_len": 32, **temperatures}
+ path = self.repo / "rl_agent_config.json"
+ path.write_text(json.dumps(cfg))
+ original = path.read_bytes()
+ with warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
+ agent = load(str(self.repo), device="cpu")
+ self.assertEqual(agent.device.type, "cpu")
+ self.assertEqual(path.read_bytes(), original)
+ # JSON comparison also handles NaN, which is unequal to itself.
+ self.assertEqual(json.dumps(agent.cfg), json.dumps(cfg))
+ self.assertEqual(json.dumps(agent.temperature_raw), json.dumps(cfg.get("temperature", [1.0] * 3)))
+ self.assertEqual(json.dumps(agent.temperature_by_options_raw), json.dumps(cfg.get("temperature_by_options", {})))
+ messages = [w for w in caught if str(w.message).startswith("laya:")]
+ self.assertTrue(all(issubclass(w.category, RuntimeWarning) for w in messages))
+ return agent, [str(w.message) for w in messages]
+
+ def assert_predictions(self, agent, choice, score, noul):
+ questions = {
+ "choice": {"type": "choice", "instructions": "Pick one", "criteria": ["a", "b"]},
+ "score": {"type": "score", "instructions": "Rate", "criteria": ["low", "mid", "high"]},
+ "noul": {"type": "noul", "instructions": "True?"},
+ }
+ # Loading is real; fixed logits make the expected scaling independent of random weights.
+ logits = torch.tensor([[0.0, 1.0, -1e4], [0.0, 1.0, 2.0], [0.0, 1.0, -1e4]])
+ with patch.object(agent.model, "forward", return_value=(logits, torch.zeros(3, 2))):
+ answers = agent.predict("hello", questions)["answers"]
+ for name, temperature, k in (("choice", choice, 2), ("score", score, 3), ("noul", noul, 2)):
+ p = torch.softmax(torch.arange(k, dtype=torch.float32) / temperature, -1)
+ if name == "noul":
+ self.assertAlmostEqual(answers[name]["noul"], p[1].item(), delta=0.0001)
+ else:
+ actual = list(answers[name]["probabilities"].values())
+ self.assertEqual(len(actual), k)
+ for got, want in zip(actual, p.tolist()):
+ self.assertAlmostEqual(got, want, delta=0.0001)
+
+ def test_invalid_type_entries_use_neutral_fallback_and_warn(self):
+ for value in (None, "invalid", "", [], {}):
+ with self.subTest(value=value):
+ agent, messages = self.load_config(temperature=[value, 2.0, 3.0])
+ self.assertEqual(agent.temperature, [1.0, 2.0, 3.0])
+ self.assertEqual(len(messages), 1)
+ self.assertIn("temperature[0]", messages[0])
+ self.assertIn(repr(value), messages[0])
+ self.assertIn("uncalibrated", messages[0])
+ self.assert_predictions(agent, choice=1.0, score=2.0, noul=3.0)
+
+ def test_invalid_bucket_entries_use_neutral_fallback_and_keep_precedence(self):
+ for value in (None, "invalid", "", [], {}):
+ with self.subTest(value=value):
+ agent, messages = self.load_config(temperature=[2.0, 3.0, 4.0],
+ temperature_by_options={"choice:2": value})
+ self.assertEqual(agent.temperature_by_options, {"choice:2": 1.0})
+ self.assertEqual(len(messages), 1)
+ self.assertIn("choice:2", messages[0])
+ self.assertIn(repr(value), messages[0])
+ self.assert_predictions(agent, choice=1.0, score=3.0, noul=4.0)
+
+ def test_nonfinite_values_keep_neutral_fallback_and_warn(self):
+ for value in (float("nan"), float("inf"), float("-inf"), "NaN", "Infinity", "-Infinity"):
+ with self.subTest(value=value):
+ agent, messages = self.load_config(temperature=[value, 2.0, 3.0],
+ temperature_by_options={"noul:2": value})
+ self.assertEqual(agent.temperature, [1.0, 2.0, 3.0])
+ self.assertEqual(agent.temperature_by_options, {"noul:2": 1.0})
+ self.assertEqual(len(messages), 1)
+ self.assertIn("temperature[0]", messages[0])
+ self.assertIn("noul:2", messages[0])
+ self.assert_predictions(agent, choice=1.0, score=2.0, noul=1.0)
+
+ def test_out_of_range_values_keep_existing_clamps(self):
+ agent, messages = self.load_config(temperature=[0.1, 10, -1],
+ temperature_by_options={"choice:2": 9, "score:3-5": 0})
+ self.assertEqual(agent.temperature, [0.5, 5.0, 0.5])
+ self.assertEqual(agent.temperature_by_options, {"choice:2": 5.0, "score:3-5": 0.5})
+ self.assertEqual(len(messages), 1)
+ for entry in ("temperature[0]", "temperature[1]", "temperature[2]", "choice:2", "score:3-5"):
+ self.assertIn(entry, messages[0])
+ self.assert_predictions(agent, choice=5.0, score=0.5, noul=0.5)
+
+ def test_valid_numbers_and_numeric_strings_do_not_warn(self):
+ agent, messages = self.load_config(temperature=[0.5, "2", 5],
+ temperature_by_options={"choice:2": "1.5", "score:3-5": 2.5, "noul:2": "5.0"})
+ self.assertEqual(agent.temperature, [0.5, 2.0, 5.0])
+ self.assertEqual(agent.temperature_by_options, {"choice:2": 1.5, "score:3-5": 2.5, "noul:2": 5.0})
+ self.assertEqual(messages, [])
+ self.assert_predictions(agent, choice=1.5, score=2.5, noul=5.0)
+
+ def test_missing_bucket_uses_corresponding_type(self):
+ agent, messages = self.load_config(temperature=[2.0, 3.0, 4.0], temperature_by_options={"choice:2": 1.5})
+ self.assertEqual(messages, [])
+ self.assert_predictions(agent, choice=1.5, score=3.0, noul=4.0)
+
+ def test_missing_temperature_fields_keep_defaults(self):
+ agent, messages = self.load_config()
+ self.assertEqual(agent.temperature, [1.0, 1.0, 1.0])
+ self.assertEqual(agent.temperature_by_options, {})
+ self.assertEqual(messages, [])
+ self.assert_predictions(agent, choice=1.0, score=1.0, noul=1.0)
+
+
+if __name__ == "__main__":
+ torch.set_num_threads(1)
+ unittest.main()