Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(train): warn when the fine-tune's calibration rests on too little

GuilhermeF669f4214906e
diff --git a/laya/train.py b/laya/train.py
index 5e861f85..5fa65add 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -17,7 +17,9 @@ copies do not have:
Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned checkpoint is
fitted with the same clamp and buckets the runtime applies instead of a local copy of the
-fitter.
+fitter. `calibration_report` says per question type what that fit rests on, and `finetune` warns
+when it rests on too little: a short run can otherwise save a checkpoint whose temperatures never
+moved from 1.0, and nothing but the numbers would show it.
Items keep the question and the tokenized state rather than a finished sequence, because a
shuffled epoch needs to rebuild the head. States are tokenized once per row and shared by
@@ -29,14 +31,18 @@ import json
import math
import os
import random
+import warnings
from dataclasses import asdict, dataclass
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
import torch
-from .calibrate import fit_temperature_map
+from .calibrate import MIN_TYPE_N, fit_temperature_map
from .common import (
+ QTYPE_NAMES,
QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
build_head,
build_model,
build_sequence,
@@ -48,6 +54,9 @@ from .common import (
)
LOSSES = ("soft-ce", "rlcd")
+# Below this many calibration items of a type, a fitted temperature is reported as resting on
+# little evidence. MIN_TYPE_N (laya.calibrate) is the floor below which it is not fitted at all.
+CALIB_WARN_N = 50
_MASKED = -1e4
@@ -452,6 +461,59 @@ def calibration_records(model, tok, items: Sequence[Dict[str, Any]], device: tor
return records
+def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],
+ temperature: Sequence[float]) -> Dict[str, Dict[str, Any]]:
+ """Per question type: how many calibration items there were, what was fitted, and what to doubt.
+
+ `records` are the `calibration_records` the temperatures were fitted on, and `temperature`
+ the per-type scalars `fit_temperature_map` returned. Each type gets `items`, `temperature`,
+ `accuracy` and `mean_confidence` (top probability at the fitted temperature) on the
+ calibration items, and a list of `issues`, empty when nothing looks wrong:
+
+ - fewer than `MIN_TYPE_N` items: the temperature was not fitted and stays 1.0;
+ - fewer than `CALIB_WARN_N` items: it was fitted, on little evidence;
+ - it landed on the `[TEMP_MIN, TEMP_MAX]` clamp;
+ - it came back unchanged at 1.0 although it was fitted, which usually means the items gave
+ the fit nothing to correct -- what a model already certain and right on them produces.
+
+ A type with no calibration items at all has no issues; it was not in the data.
+ """
+ import numpy as np
+
+ by_type: Dict[int, List[Tuple[Any, Any]]] = {qt: [] for qt in QTYPE_NAMES}
+ for qt, logits, target, _k in records:
+ by_type[int(qt)].append((np.asarray(logits, dtype=float), np.asarray(target, dtype=float)))
+ report = {}
+ for qt, name in QTYPE_NAMES.items():
+ pairs, t = by_type[qt], float(temperature[qt])
+ entry: Dict[str, Any] = {"items": len(pairs), "temperature": t, "issues": []}
+ if pairs:
+ right, conf = 0, 0.0
+ for logits, target in pairs:
+ z = logits / t
+ p = np.exp(z - z.max())
+ p /= p.sum()
+ right += int(p.argmax() == target.argmax())
+ conf += float(p.max())
+ entry["accuracy"] = right / len(pairs)
+ entry["mean_confidence"] = conf / len(pairs)
+ n, issues = len(pairs), entry["issues"]
+ if 0 < n < MIN_TYPE_N:
+ issues.append("not fitted: %d calibration items, fewer than %d, so the temperature stays 1.0"
+ % (n, MIN_TYPE_N))
+ elif n >= MIN_TYPE_N:
+ if n < CALIB_WARN_N:
+ issues.append("fitted on only %d calibration items" % n)
+ if t <= TEMP_MIN or t >= TEMP_MAX:
+ issues.append("the fitted temperature %.3g is on the [%g, %g] clamp" % (t, TEMP_MIN, TEMP_MAX))
+ elif abs(t - 1.0) < 1e-3:
+ issues.append("the fit came back unchanged at 1.0 (accuracy %.2f, mean confidence %.3f on the "
+ "calibration items), which usually means they gave it nothing to correct; this "
+ "type's confidences are uncalibrated" % (entry["accuracy"], entry["mean_confidence"]))
+ report[name] = entry
+ return report
+
+
def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,
device: Optional[str] = "auto") -> Dict[str, Any]:
"""Preprocess, train, calibrate and save; returns a summary of the run.
@@ -481,14 +543,22 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
history = train_model(model, tok, train_items, config, dev, max_len, head_max_len,
on_epoch_end=checkpoint_latest)
- fitted = fit_temperature_map(calibration_records(model, tok, calib_items, dev, max_len, head_max_len))
+ records = calibration_records(model, tok, calib_items, dev, max_len, head_max_len)
+ fitted = fit_temperature_map(records)
+ calibration = calibration_report(records, fitted["temperature"])
+ for name, entry in calibration.items():
+ for issue in entry["issues"]:
+ # Logged as a warning rather than left to the report alone: a checkpoint whose
+ # confidences were never calibrated looks the same as one that was.
+ warnings.warn("laya.train: %s calibration: %s" % (name, issue), RuntimeWarning, stacklevel=2)
out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,
temperature=fitted["temperature"])
# An inherited bucket map takes precedence at inference and would mask the new fit.
out_cfg.pop("temperature_by_options", None)
if fitted["temperature_by_options"]:
out_cfg["temperature_by_options"] = fitted["temperature_by_options"]
- out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))
+ out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config),
+ laya_train_calibration=calibration)
save_checkpoint(model, tok, out_cfg, output_dir)
return {
"train_items": len(train_items),
@@ -498,5 +568,6 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
"temperature": fitted["temperature"],
"temperature_by_options": fitted["temperature_by_options"],
"n_by_bucket": fitted["n_by_bucket"],
+ "calibration": calibration,
"output_dir": output_dir,
}
diff --git a/tests/test_train.py b/tests/test_train.py
index f7ca5558..bb633639 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -9,9 +9,11 @@ import json
import os
from pathlib import Path
import random
+import re
import sys
import tempfile
import unittest
+import warnings
from unittest.mock import patch
os.environ.setdefault("USE_TF", "0")
@@ -392,5 +394,59 @@ class EndToEndTests(unittest.TestCase):
train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)
+class CalibrationReportTests(unittest.TestCase):
+ @staticmethod
+ def records(qtype, n, k=3, scale=1.0):
+ # Logits that favour the target option by `scale`; target one-hot on option 0.
+ return [(qtype, [scale] + [0.0] * (k - 1), [1.0] + [0.0] * (k - 1), k) for _ in range(n)]
+
+ def test_each_kind_of_weak_fit_is_named(self):
+ from laya.train import CALIB_WARN_N, calibration_report
+
+ records = (self.records(QTYPES["choice"], 5) + self.records(QTYPES["noul"], 30, k=2)
+ + self.records(QTYPES["score"], CALIB_WARN_N + 10))
+ report = calibration_report(records, [1.0, 1.0, 2.0]) # choice, score, noul
+ self.assertEqual(report["choice"]["items"], 5)
+ self.assertIn("not fitted", report["choice"]["issues"][0])
+ self.assertEqual(len(report["noul"]["issues"]), 1)
+ self.assertIn("only 30", report["noul"]["issues"][0])
+ self.assertEqual(len(report["score"]["issues"]), 1)
+ self.assertIn("unchanged at 1.0", report["score"]["issues"][0])
+ self.assertAlmostEqual(report["score"]["accuracy"], 1.0)
+
+ def test_clamped_and_healthy_fits(self):
+ from laya.train import CALIB_WARN_N, calibration_report
+
+ n = CALIB_WARN_N
+ records = self.records(QTYPES["choice"], n) + self.records(QTYPES["score"], n)
+ report = calibration_report(records, [TEMP_MIN, 1.3, 1.0]) # choice, score, noul
+ self.assertIn("clamp", report["choice"]["issues"][0])
+ self.assertEqual(report["score"]["issues"], [])
+ # a type absent from the data is reported, but not as a problem
+ self.assertEqual((report["noul"]["items"], report["noul"]["issues"]), (0, []))
+ self.assertNotIn("accuracy", report["noul"])
+ p = torch.softmax(torch.tensor([1.0, 0.0, 0.0]) / 1.3, -1)
+ self.assertAlmostEqual(report["score"]["mean_confidence"], p.max().item(), places=6)
+
+ def test_finetune_warns_and_records_a_weak_calibration(self):
+ with tempfile.TemporaryDirectory() as d:
+ root = Path(d)
+ make_checkpoint(root / "base", make_tokenizer())
+ data = root / "train.jsonl"
+ data.write_text("\n".join(json.dumps(r) for r in rows(2)), encoding="utf-8")
+ config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)
+ # warnings.catch_warnings, not assertWarns: assertWarns walks every loaded module, and
+ # transformers' lazy modules then try to import optional packages such as torchvision.
+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")), \
+ warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ report = finetune(str(data), str(root / "base"), str(root / "out"), config, device="cpu")
+ saved = json.loads((root / "out" / "rl_agent_config.json").read_text())
+ messages = [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]
+ self.assertTrue(any(re.match(r"laya\.train: \w+ calibration: not fitted", m) for m in messages), messages)
+ self.assertEqual(saved["training"]["laya_train_calibration"], report["calibration"])
+ self.assertEqual(sum(e["items"] for e in report["calibration"].values()), report["calibration_items"])
+
+
if __name__ == "__main__":
unittest.main()