Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(train): warn when the fine-tune's calibration rests on too little
GuilhermeF669f4214906e
diff --git a/laya/train.py b/laya/train.pyindex 5e861f85..5fa65add 100644--- a/laya/train.py+++ b/laya/train.py@@ -17,7 +17,9 @@ copies do not have:Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned checkpoint isfitted with the same clamp and buckets the runtime applies instead of a local copy of the-fitter.+fitter. `calibration_report` says per question type what that fit rests on, and `finetune` warns+when it rests on too little: a short run can otherwise save a checkpoint whose temperatures never+moved from 1.0, and nothing but the numbers would show it.Items keep the question and the tokenized state rather than a finished sequence, because ashuffled epoch needs to rebuild the head. States are tokenized once per row and shared by@@ -29,14 +31,18 @@ import jsonimport mathimport osimport random+import warningsfrom dataclasses import asdict, dataclassfrom typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tupleimport torch-from .calibrate import fit_temperature_map+from .calibrate import MIN_TYPE_N, fit_temperature_mapfrom .common import (+ QTYPE_NAMES,QTYPES,+ TEMP_MAX,+ TEMP_MIN,build_head,build_model,build_sequence,@@ -48,6 +54,9 @@ from .common import ()LOSSES = ("soft-ce", "rlcd")+# Below this many calibration items of a type, a fitted temperature is reported as resting on+# little evidence. MIN_TYPE_N (laya.calibrate) is the floor below which it is not fitted at all.+CALIB_WARN_N = 50_MASKED = -1e4@@ -452,6 +461,59 @@ def calibration_records(model, tok, items: Sequence[Dict[str, Any]], device: torreturn records+def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],+ temperature: Sequence[float]) -> Dict[str, Dict[str, Any]]:+ """Per question type: how many calibration items there were, what was fitted, and what to doubt.++ `records` are the `calibration_records` the temperatures were fitted on, and `temperature`+ the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature`,+ `accuracy` and `mean_confidence` (top probability at the fitted temperature) on the+ calibration items, and a list of `issues`, empty when nothing looks wrong:++ - fewer than `MIN_TYPE_N` items: the temperature was not fitted and stays 1.0;+ - fewer than `CALIB_WARN_N` items: it was fitted, on little evidence;+ - it landed on the `[TEMP_MIN, TEMP_MAX]` clamp;+ - it came back unchanged at 1.0 although it was fitted, which usually means the items gave+ the fit nothing to correct -- what a model already certain and right on them produces.++ A type with no calibration items at all has no issues; it was not in the data.+ """+ import numpy as np++ by_type: Dict[int, List[Tuple[Any, Any]]] = {qt: [] for qt in QTYPE_NAMES}+ for qt, logits, target, _k in records:+ by_type[int(qt)].append((np.asarray(logits, dtype=float), np.asarray(target, dtype=float)))+ report = {}+ for qt, name in QTYPE_NAMES.items():+ pairs, t = by_type[qt], float(temperature[qt])+ entry: Dict[str, Any] = {"items": len(pairs), "temperature": t, "issues": []}+ if pairs:+ right, conf = 0, 0.0+ for logits, target in pairs:+ z = logits / t+ p = np.exp(z - z.max())+ p /= p.sum()+ right += int(p.argmax() == target.argmax())+ conf += float(p.max())+ entry["accuracy"] = right / len(pairs)+ entry["mean_confidence"] = conf / len(pairs)+ n, issues = len(pairs), entry["issues"]+ if 0 < n < MIN_TYPE_N:+ issues.append("not fitted: %d calibration items, fewer than %d, so the temperature stays 1.0"+ % (n, MIN_TYPE_N))+ elif n >= MIN_TYPE_N:+ if n < CALIB_WARN_N:+ issues.append("fitted on only %d calibration items" % n)+ if t <= TEMP_MIN or t >= TEMP_MAX:+ issues.append("the fitted temperature %.3g is on the [%g, %g] clamp" % (t, TEMP_MIN, TEMP_MAX))+ elif abs(t - 1.0) < 1e-3:+ issues.append("the fit came back unchanged at 1.0 (accuracy %.2f, mean confidence %.3f on the "+ "calibration items), which usually means they gave it nothing to correct; this "+ "type's confidences are uncalibrated" % (entry["accuracy"], entry["mean_confidence"]))+ report[name] = entry+ return report++def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,device: Optional[str] = "auto") -> Dict[str, Any]:"""Preprocess, train, calibrate and save; returns a summary of the run.@@ -481,14 +543,22 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainChistory = train_model(model, tok, train_items, config, dev, max_len, head_max_len,on_epoch_end=checkpoint_latest)- fitted = fit_temperature_map(calibration_records(model, tok, calib_items, dev, max_len, head_max_len))+ records = calibration_records(model, tok, calib_items, dev, max_len, head_max_len)+ fitted = fit_temperature_map(records)+ calibration = calibration_report(records, fitted["temperature"])+ for name, entry in calibration.items():+ for issue in entry["issues"]:+ # Logged as a warning rather than left to the report alone: a checkpoint whose+ # confidences were never calibrated looks the same as one that was.+ warnings.warn("laya.train: %s calibration: %s" % (name, issue), RuntimeWarning, stacklevel=2)out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,temperature=fitted["temperature"])# An inherited bucket map takes precedence at inference and would mask the new fit.out_cfg.pop("temperature_by_options", None)if fitted["temperature_by_options"]:out_cfg["temperature_by_options"] = fitted["temperature_by_options"]- out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))+ out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config),+ laya_train_calibration=calibration)save_checkpoint(model, tok, out_cfg, output_dir)return {"train_items": len(train_items),@@ -498,5 +568,6 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC"temperature": fitted["temperature"],"temperature_by_options": fitted["temperature_by_options"],"n_by_bucket": fitted["n_by_bucket"],+ "calibration": calibration,"output_dir": output_dir,}diff --git a/tests/test_train.py b/tests/test_train.pyindex f7ca5558..bb633639 100644--- a/tests/test_train.py+++ b/tests/test_train.py@@ -9,9 +9,11 @@ import jsonimport osfrom pathlib import Pathimport random+import reimport sysimport tempfileimport unittest+import warningsfrom unittest.mock import patchos.environ.setdefault("USE_TF", "0")@@ -392,5 +394,59 @@ class EndToEndTests(unittest.TestCase):train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)+class CalibrationReportTests(unittest.TestCase):+ @staticmethod+ def records(qtype, n, k=3, scale=1.0):+ # Logits that favour the target option by `scale`; target one-hot on option 0.+ return [(qtype, [scale] + [0.0] * (k - 1), [1.0] + [0.0] * (k - 1), k) for _ in range(n)]++ def test_each_kind_of_weak_fit_is_named(self):+ from laya.train import CALIB_WARN_N, calibration_report++ records = (self.records(QTYPES["choice"], 5) + self.records(QTYPES["noul"], 30, k=2)+ + self.records(QTYPES["score"], CALIB_WARN_N + 10))+ report = calibration_report(records, [1.0, 1.0, 2.0]) # choice, score, noul+ self.assertEqual(report["choice"]["items"], 5)+ self.assertIn("not fitted", report["choice"]["issues"][0])+ self.assertEqual(len(report["noul"]["issues"]), 1)+ self.assertIn("only 30", report["noul"]["issues"][0])+ self.assertEqual(len(report["score"]["issues"]), 1)+ self.assertIn("unchanged at 1.0", report["score"]["issues"][0])+ self.assertAlmostEqual(report["score"]["accuracy"], 1.0)++ def test_clamped_and_healthy_fits(self):+ from laya.train import CALIB_WARN_N, calibration_report++ n = CALIB_WARN_N+ records = self.records(QTYPES["choice"], n) + self.records(QTYPES["score"], n)+ report = calibration_report(records, [TEMP_MIN, 1.3, 1.0]) # choice, score, noul+ self.assertIn("clamp", report["choice"]["issues"][0])+ self.assertEqual(report["score"]["issues"], [])+ # a type absent from the data is reported, but not as a problem+ self.assertEqual((report["noul"]["items"], report["noul"]["issues"]), (0, []))+ self.assertNotIn("accuracy", report["noul"])+ p = torch.softmax(torch.tensor([1.0, 0.0, 0.0]) / 1.3, -1)+ self.assertAlmostEqual(report["score"]["mean_confidence"], p.max().item(), places=6)++ def test_finetune_warns_and_records_a_weak_calibration(self):+ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ make_checkpoint(root / "base", make_tokenizer())+ data = root / "train.jsonl"+ data.write_text("\n".join(json.dumps(r) for r in rows(2)), encoding="utf-8")+ config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)+ # warnings.catch_warnings, not assertWarns: assertWarns walks every loaded module, and+ # transformers' lazy modules then try to import optional packages such as torchvision.+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")), \+ warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ report = finetune(str(data), str(root / "base"), str(root / "out"), config, device="cpu")+ saved = json.loads((root / "out" / "rl_agent_config.json").read_text())+ messages = [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]+ self.assertTrue(any(re.match(r"laya\.train: \w+ calibration: not fitted", m) for m in messages), messages)+ self.assertEqual(saved["training"]["laya_train_calibration"], report["calibration"])+ self.assertEqual(sum(e["items"] for e in report["calibration"].values()), report["calibration_items"])++if __name__ == "__main__":unittest.main()