Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(train): warn when fine-tune predictions collapse to the class prior
Tiago Vilas Boas923ff821662f
diff --git a/laya/train.py b/laya/train.pyindex cca4e6a0..6bf3b0fb 100644--- a/laya/train.py+++ b/laya/train.py@@ -19,7 +19,10 @@ Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned cfitted with the same clamp and buckets the runtime applies instead of a local copy of thefitter. `calibration_report` says per question type what that fit rests on, and `finetune` warnswhen it rests on too little: a short run can otherwise save a checkpoint whose temperatures never-moved from 1.0, and nothing but the numbers would show it.+moved from 1.0, and nothing but the numbers would show it. It also warns when predictions have+collapsed to the class prior -- a near-constant logit (or calibration CE stuck at ln K) that+looks like a finished run on an unlearnable task, but is often just too few updates on a small+dataset (#963).Items keep the question and the tokenized state rather than a finished sequence, because ashuffled epoch needs to rebuild the head. States are tokenized once per row and shared by@@ -59,6 +62,13 @@ LOSSES = ("soft-ce", "rlcd")# Below this many calibration items of a type, a fitted temperature is reported as resting on# little evidence. MIN_TYPE_N (laya.calibrate) is the floor below which it is not fitted at all.CALIB_WARN_N = 50+# Mean within-row logit range below this is the silent collapse to a constant (usually+# the class prior) measured in #963: 0.01 on a collapsed head vs 0.62 untuned.+COLLAPSE_LOGIT_RANGE = 0.05+# Calibration mean cross-entropy within this relative distance of ln K is the other+# collapse signal from #963 (training CE stuck at ln 4). The reported training loss is+# not used: the default `rlcd` objective is not cross-entropy.+COLLAPSE_CE_REL = 0.02_MASKED = -1e4@@ -753,6 +763,69 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],return report+def collapse_stats(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[Dict[str, float]]:+ """Mean within-row logit range, mean soft-CE and mean ln K on records with k >= 2.++ A 1-option row cannot collapse to a prior, so it is skipped. Empty input, or only+ 1-option rows, returns None.+ """+ import numpy as np++ ranges, ces, lnks = [], [], []+ for _qt, logits, target, k in records:+ k = int(k)+ if k < 2:+ continue+ z = np.asarray(logits, dtype=float).ravel()[:k]+ t = np.asarray(target, dtype=float).ravel()[:k]+ mass = float(t.sum())+ if mass <= 0.0:+ continue+ t = t / mass+ ranges.append(float(z.max() - z.min()))+ logp = z - np.logaddexp.reduce(z)+ ces.append(float(-(t * logp).sum()))+ lnks.append(math.log(k))+ if not ranges:+ return None+ n = len(ranges)+ return {+ "n": float(n),+ "mean_logit_range": sum(ranges) / n,+ "mean_ce": sum(ces) / n,+ "mean_ln_k": sum(lnks) / n,+ }+++def prior_collapse_message(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[str]:+ """Warning text if calibration logits have collapsed to the class prior, else None.++ The default 4-epoch budget is sized for a few thousand typed decisions. On a few+ hundred or a thousand labelled rows the head can finish with every option at the+ same logit -- chance-level, and silent. #963 measured a mean within-row range of+ 0.01 against 0.62 for the untuned checkpoint, and training CE stuck at ln K.++ Fired when the mean within-row logit range is below `COLLAPSE_LOGIT_RANGE`, or when+ mean soft-CE on the same records is within `COLLAPSE_CE_REL` of mean ln K.+ """+ stats = collapse_stats(records)+ if stats is None:+ return None+ reasons = []+ if stats["mean_logit_range"] < COLLAPSE_LOGIT_RANGE:+ reasons.append("mean within-row logit range %.3g" % stats["mean_logit_range"])+ mean_ce, mean_lnk = stats["mean_ce"], stats["mean_ln_k"]+ if mean_lnk > 0.0 and abs(mean_ce - mean_lnk) <= COLLAPSE_CE_REL * mean_lnk:+ reasons.append("calibration cross-entropy %.3f is within %.0f%% of ln K ~ %.3f"+ % (mean_ce, COLLAPSE_CE_REL * 100.0, mean_lnk))+ if not reasons:+ return None+ return ("laya.train: predictions collapsed to the class prior (%s). "+ "The default epoch budget is often too small on a few hundred or a thousand "+ "labelled rows. Try more epochs, more data, or another seed."+ % "; ".join(reasons))++def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,device: Optional[str] = "auto") -> Dict[str, Any]:"""Preprocess, train, calibrate and save; returns a summary of the run.@@ -790,6 +863,11 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCon_epoch_end=checkpoint_latest, parallel=parallel)records = calibration_records(model, tok, calib_items, dev, max_len, head_max_len, parallel=parallel)+ collapse_msg = prior_collapse_message(records)+ if collapse_msg:+ # Same channel as the weak-calibration notes: a checkpoint that predicts the+ # class prior for every input otherwise looks like a finished, unlearnable task.+ warnings.warn(collapse_msg, RuntimeWarning, stacklevel=2)fitted = fit_temperature_map(records)calibration = calibration_report(records, fitted["temperature"])for name, entry in calibration.items():diff --git a/tests/test_train.py b/tests/test_train.pyindex bd2ee6a7..87c46b19 100644--- a/tests/test_train.py+++ b/tests/test_train.py@@ -6,6 +6,7 @@ in a temp directory, so nothing is downloaded.Run: python tests/test_train.py"""import json+import mathimport osfrom pathlib import Pathimport random@@ -709,5 +710,80 @@ class CalibrationReportTests(unittest.TestCase):self.assertEqual(sum(e["items"] for e in report["calibration"].values()), report["calibration_items"])+class PriorCollapseTests(unittest.TestCase):+ """#963: a collapsed head is chance-level and silent unless finetune warns."""++ @staticmethod+ def rows_of(logits, target, n=8, qtype=None, k=None):+ qtype = QTYPES["choice"] if qtype is None else qtype+ k = len(logits) if k is None else k+ return [(qtype, list(logits), list(target), k) for _ in range(n)]++ def test_constant_logits_match_the_prior_and_warn(self):+ from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message++ # Tiny 4-way set: every row is the same logit, so the predicted distribution is the+ # uniform prior. This is the measured failure in #963 (range 0.01, CE at ln 4).+ records = self.rows_of([0.002, 0.0, -0.001, 0.001], [1.0, 0.0, 0.0, 0.0])+ stats = collapse_stats(records)+ self.assertIsNotNone(stats)+ self.assertLess(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)+ self.assertAlmostEqual(stats["mean_ce"], math.log(4), places=2)+ msg = prior_collapse_message(records)+ self.assertIsNotNone(msg)+ self.assertIn("collapsed to the class prior", msg)+ self.assertIn("more epochs", msg)+ self.assertIn("more data", msg)+ self.assertIn("another seed", msg)++ def test_peaked_logits_on_a_learnable_set_do_not_warn(self):+ from laya.train import prior_collapse_message++ records = self.rows_of([5.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0])+ self.assertIsNone(prior_collapse_message(records))+ self.assertIsNone(prior_collapse_message([]))+ self.assertIsNone(prior_collapse_message(self.rows_of([0.0], [1.0], k=1)))++ def test_ce_stuck_at_ln_k_warns_even_when_logit_range_is_not_tiny(self):+ from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message++ # Softmax is still near-uniform, so CE sits on ln K, but the within-row range is+ # above the constant-logit cutoff -- the other #963 trigger.+ records = self.rows_of([0.2, 0.05, -0.05, -0.2], [0.25, 0.25, 0.25, 0.25])+ stats = collapse_stats(records)+ self.assertGreaterEqual(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)+ self.assertAlmostEqual(stats["mean_ce"], stats["mean_ln_k"], delta=0.03)+ msg = prior_collapse_message(records)+ self.assertIsNotNone(msg)+ self.assertIn("cross-entropy", msg)++ def test_finetune_emits_the_collapse_warning_only_when_logits_collapse(self):+ from laya.train import prior_collapse_message++ collapsed = self.rows_of([0.01, 0.0, -0.01], [1.0, 0.0, 0.0], n=12)+ peaked = self.rows_of([4.0, 0.0, 0.0], [1.0, 0.0, 0.0], n=12)+ self.assertIsNotNone(prior_collapse_message(collapsed))+ self.assertIsNone(prior_collapse_message(peaked))++ def run(records):+ with tempfile.TemporaryDirectory() as d:+ root = Path(d)+ make_checkpoint(root / "base", make_tokenizer())+ config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)+ with patch("huggingface_hub.snapshot_download",+ side_effect=AssertionError("unexpected download")), \+ patch("laya.train.read_jsonl", return_value=rows(2)), \+ patch("laya.train.calibration_records", return_value=records), \+ warnings.catch_warnings(record=True) as caught:+ warnings.simplefilter("always")+ finetune("rows.jsonl", str(root / "base"), str(root / "out"), config, device="cpu")+ return [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]++ collapsed_msgs = run(collapsed)+ self.assertTrue(any("collapsed to the class prior" in m for m in collapsed_msgs), collapsed_msgs)+ peaked_msgs = run(peaked)+ self.assertFalse(any("collapsed to the class prior" in m for m in peaked_msgs), peaked_msgs)++if __name__ == "__main__":unittest.main()