Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(train): warn when fine-tune predictions collapse to the class prior

Tiago Vilas Boas923ff821662f
diff --git a/laya/train.py b/laya/train.py
index cca4e6a0..6bf3b0fb 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -19,7 +19,10 @@ Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned c
fitted with the same clamp and buckets the runtime applies instead of a local copy of the
fitter. `calibration_report` says per question type what that fit rests on, and `finetune` warns
when it rests on too little: a short run can otherwise save a checkpoint whose temperatures never
-moved from 1.0, and nothing but the numbers would show it.
+moved from 1.0, and nothing but the numbers would show it. It also warns when predictions have
+collapsed to the class prior -- a near-constant logit (or calibration CE stuck at ln K) that
+looks like a finished run on an unlearnable task, but is often just too few updates on a small
+dataset (#963).
Items keep the question and the tokenized state rather than a finished sequence, because a
shuffled epoch needs to rebuild the head. States are tokenized once per row and shared by
@@ -59,6 +62,13 @@ LOSSES = ("soft-ce", "rlcd")
# Below this many calibration items of a type, a fitted temperature is reported as resting on
# little evidence. MIN_TYPE_N (laya.calibrate) is the floor below which it is not fitted at all.
CALIB_WARN_N = 50
+# Mean within-row logit range below this is the silent collapse to a constant (usually
+# the class prior) measured in #963: 0.01 on a collapsed head vs 0.62 untuned.
+COLLAPSE_LOGIT_RANGE = 0.05
+# Calibration mean cross-entropy within this relative distance of ln K is the other
+# collapse signal from #963 (training CE stuck at ln 4). The reported training loss is
+# not used: the default `rlcd` objective is not cross-entropy.
+COLLAPSE_CE_REL = 0.02
_MASKED = -1e4
@@ -753,6 +763,69 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],
return report
+def collapse_stats(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[Dict[str, float]]:
+ """Mean within-row logit range, mean soft-CE and mean ln K on records with k >= 2.
+
+ A 1-option row cannot collapse to a prior, so it is skipped. Empty input, or only
+ 1-option rows, returns None.
+ """
+ import numpy as np
+
+ ranges, ces, lnks = [], [], []
+ for _qt, logits, target, k in records:
+ k = int(k)
+ if k < 2:
+ continue
+ z = np.asarray(logits, dtype=float).ravel()[:k]
+ t = np.asarray(target, dtype=float).ravel()[:k]
+ mass = float(t.sum())
+ if mass <= 0.0:
+ continue
+ t = t / mass
+ ranges.append(float(z.max() - z.min()))
+ logp = z - np.logaddexp.reduce(z)
+ ces.append(float(-(t * logp).sum()))
+ lnks.append(math.log(k))
+ if not ranges:
+ return None
+ n = len(ranges)
+ return {
+ "n": float(n),
+ "mean_logit_range": sum(ranges) / n,
+ "mean_ce": sum(ces) / n,
+ "mean_ln_k": sum(lnks) / n,
+ }
+
+
+def prior_collapse_message(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[str]:
+ """Warning text if calibration logits have collapsed to the class prior, else None.
+
+ The default 4-epoch budget is sized for a few thousand typed decisions. On a few
+ hundred or a thousand labelled rows the head can finish with every option at the
+ same logit -- chance-level, and silent. #963 measured a mean within-row range of
+ 0.01 against 0.62 for the untuned checkpoint, and training CE stuck at ln K.
+
+ Fired when the mean within-row logit range is below `COLLAPSE_LOGIT_RANGE`, or when
+ mean soft-CE on the same records is within `COLLAPSE_CE_REL` of mean ln K.
+ """
+ stats = collapse_stats(records)
+ if stats is None:
+ return None
+ reasons = []
+ if stats["mean_logit_range"] < COLLAPSE_LOGIT_RANGE:
+ reasons.append("mean within-row logit range %.3g" % stats["mean_logit_range"])
+ mean_ce, mean_lnk = stats["mean_ce"], stats["mean_ln_k"]
+ if mean_lnk > 0.0 and abs(mean_ce - mean_lnk) <= COLLAPSE_CE_REL * mean_lnk:
+ reasons.append("calibration cross-entropy %.3f is within %.0f%% of ln K ~ %.3f"
+ % (mean_ce, COLLAPSE_CE_REL * 100.0, mean_lnk))
+ if not reasons:
+ return None
+ return ("laya.train: predictions collapsed to the class prior (%s). "
+ "The default epoch budget is often too small on a few hundred or a thousand "
+ "labelled rows. Try more epochs, more data, or another seed."
+ % "; ".join(reasons))
+
+
def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,
device: Optional[str] = "auto") -> Dict[str, Any]:
"""Preprocess, train, calibrate and save; returns a summary of the run.
@@ -790,6 +863,11 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
on_epoch_end=checkpoint_latest, parallel=parallel)
records = calibration_records(model, tok, calib_items, dev, max_len, head_max_len, parallel=parallel)
+ collapse_msg = prior_collapse_message(records)
+ if collapse_msg:
+ # Same channel as the weak-calibration notes: a checkpoint that predicts the
+ # class prior for every input otherwise looks like a finished, unlearnable task.
+ warnings.warn(collapse_msg, RuntimeWarning, stacklevel=2)
fitted = fit_temperature_map(records)
calibration = calibration_report(records, fitted["temperature"])
for name, entry in calibration.items():
diff --git a/tests/test_train.py b/tests/test_train.py
index bd2ee6a7..87c46b19 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -6,6 +6,7 @@ in a temp directory, so nothing is downloaded.
Run: python tests/test_train.py
"""
import json
+import math
import os
from pathlib import Path
import random
@@ -709,5 +710,80 @@ class CalibrationReportTests(unittest.TestCase):
self.assertEqual(sum(e["items"] for e in report["calibration"].values()), report["calibration_items"])
+class PriorCollapseTests(unittest.TestCase):
+ """#963: a collapsed head is chance-level and silent unless finetune warns."""
+
+ @staticmethod
+ def rows_of(logits, target, n=8, qtype=None, k=None):
+ qtype = QTYPES["choice"] if qtype is None else qtype
+ k = len(logits) if k is None else k
+ return [(qtype, list(logits), list(target), k) for _ in range(n)]
+
+ def test_constant_logits_match_the_prior_and_warn(self):
+ from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message
+
+ # Tiny 4-way set: every row is the same logit, so the predicted distribution is the
+ # uniform prior. This is the measured failure in #963 (range 0.01, CE at ln 4).
+ records = self.rows_of([0.002, 0.0, -0.001, 0.001], [1.0, 0.0, 0.0, 0.0])
+ stats = collapse_stats(records)
+ self.assertIsNotNone(stats)
+ self.assertLess(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)
+ self.assertAlmostEqual(stats["mean_ce"], math.log(4), places=2)
+ msg = prior_collapse_message(records)
+ self.assertIsNotNone(msg)
+ self.assertIn("collapsed to the class prior", msg)
+ self.assertIn("more epochs", msg)
+ self.assertIn("more data", msg)
+ self.assertIn("another seed", msg)
+
+ def test_peaked_logits_on_a_learnable_set_do_not_warn(self):
+ from laya.train import prior_collapse_message
+
+ records = self.rows_of([5.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0])
+ self.assertIsNone(prior_collapse_message(records))
+ self.assertIsNone(prior_collapse_message([]))
+ self.assertIsNone(prior_collapse_message(self.rows_of([0.0], [1.0], k=1)))
+
+ def test_ce_stuck_at_ln_k_warns_even_when_logit_range_is_not_tiny(self):
+ from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message
+
+ # Softmax is still near-uniform, so CE sits on ln K, but the within-row range is
+ # above the constant-logit cutoff -- the other #963 trigger.
+ records = self.rows_of([0.2, 0.05, -0.05, -0.2], [0.25, 0.25, 0.25, 0.25])
+ stats = collapse_stats(records)
+ self.assertGreaterEqual(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)
+ self.assertAlmostEqual(stats["mean_ce"], stats["mean_ln_k"], delta=0.03)
+ msg = prior_collapse_message(records)
+ self.assertIsNotNone(msg)
+ self.assertIn("cross-entropy", msg)
+
+ def test_finetune_emits_the_collapse_warning_only_when_logits_collapse(self):
+ from laya.train import prior_collapse_message
+
+ collapsed = self.rows_of([0.01, 0.0, -0.01], [1.0, 0.0, 0.0], n=12)
+ peaked = self.rows_of([4.0, 0.0, 0.0], [1.0, 0.0, 0.0], n=12)
+ self.assertIsNotNone(prior_collapse_message(collapsed))
+ self.assertIsNone(prior_collapse_message(peaked))
+
+ def run(records):
+ with tempfile.TemporaryDirectory() as d:
+ root = Path(d)
+ make_checkpoint(root / "base", make_tokenizer())
+ config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)
+ with patch("huggingface_hub.snapshot_download",
+ side_effect=AssertionError("unexpected download")), \
+ patch("laya.train.read_jsonl", return_value=rows(2)), \
+ patch("laya.train.calibration_records", return_value=records), \
+ warnings.catch_warnings(record=True) as caught:
+ warnings.simplefilter("always")
+ finetune("rows.jsonl", str(root / "base"), str(root / "out"), config, device="cpu")
+ return [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]
+
+ collapsed_msgs = run(collapsed)
+ self.assertTrue(any("collapsed to the class prior" in m for m in collapsed_msgs), collapsed_msgs)
+ peaked_msgs = run(peaked)
+ self.assertFalse(any("collapsed to the class prior" in m for m in peaked_msgs), peaked_msgs)
+
+
if __name__ == "__main__":
unittest.main()