Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #968 from tiagovilasboas/fix/train-prior-collapse-warning

Nandakishore274b6ec8962
diff --cc laya/train.py
index 7d85dfdf,6bf3b0fb..facf1cfb
--- a/laya/train.py
+++ b/laya/train.py
@@@ -764,125 -763,69 +774,188 @@@ def calibration_report(records: Sequenc
return report
+def evaluate_records(records: Sequence[Tuple[int, Any, Any, int]],
+ temperature: Optional[Sequence[float]] = None,
+ temperature_by_options: Optional[Dict[str, float]] = None) -> Dict[str, Any]:
+ """Evaluate loss, accuracy, top-1 confidence, ECE and Brier scores on `records`.
+
+ `records` is a sequence of `(qtype, logits, target, k)` tuples as returned by
+ `calibration_records`. Predictions and confidences are scaled using `temperature` /
+ `temperature_by_options` if provided. Returns overall metrics and per-question-type
+ breakdowns.
+
+ `brier` is the multi-class Brier score over the full probability vector:
+ `mean(sum((p - target)**2))`. `brier_top1` is the binary Brier score of top-1
+ confidence as P(correct): `mean((max(p) - correct)**2)`.
+ """
+ import numpy as np
+
+ if not records:
+ return {
+ "items": 0,
+ "loss": 0.0,
+ "accuracy": 0.0,
+ "mean_confidence": 0.0,
+ "ece": None,
+ "brier": None,
+ "brier_top1": None,
+ "by_type": {},
+ }
+
+ losses: List[float] = []
+ corrects: List[bool] = []
+ confs: List[float] = []
+ briers_mc: List[float] = []
+ briers_top1: List[float] = []
+ by_type: Dict[int, Dict[str, List[Any]]] = {
+ qt: {"losses": [], "corrects": [], "confs": [], "briers_mc": [], "briers_top1": []} for qt in QTYPE_NAMES
+ }
+
+ for qt, logits, target, k in records:
+ z = np.asarray(logits[:k], dtype=float)
+ t = np.asarray(target[:k], dtype=float)
+ bucket = temp_bucket(qt, k)
+ if temperature_by_options and bucket in temperature_by_options:
+ scale = float(temperature_by_options[bucket])
+ elif temperature is not None and isinstance(temperature, (list, tuple)) and qt < len(temperature):
+ scale = float(temperature[qt])
+ elif isinstance(temperature, (int, float)):
+ scale = float(temperature)
+ else:
+ scale = 1.0
+ scale = max(scale, 1e-4)
+ z_scaled = z / scale
+ z_max = np.max(z_scaled)
+ exp_z = np.exp(z_scaled - z_max)
+ sum_exp_z = np.sum(exp_z)
+ p = exp_z / sum_exp_z
+ log_p = z_scaled - z_max - np.log(sum_exp_z)
+ loss = float(-np.sum(t * log_p))
+
+ pred = int(np.argmax(p))
+ gold = int(np.argmax(t))
+ correct = bool(pred == gold)
+ conf = float(np.max(p))
+ b_mc = float(np.sum((p - t) ** 2))
+ b_top1 = float((conf - float(correct)) ** 2)
+
+ losses.append(loss)
+ corrects.append(correct)
+ confs.append(conf)
+ briers_mc.append(b_mc)
+ briers_top1.append(b_top1)
+
+ if qt in by_type:
+ by_type[qt]["losses"].append(loss)
+ by_type[qt]["corrects"].append(correct)
+ by_type[qt]["confs"].append(conf)
+ by_type[qt]["briers_mc"].append(b_mc)
+ by_type[qt]["briers_top1"].append(b_top1)
+
+ total_items = len(records)
+ acc = float(np.mean(corrects)) if corrects else 0.0
+ mean_loss = float(np.mean(losses)) if losses else 0.0
+ mean_conf = float(np.mean(confs)) if confs else 0.0
+ ece_val = float(ece_score(np.asarray(confs, dtype=float), np.asarray(corrects, dtype=bool))) if len(corrects) >= 2 else None
+ brier_val = float(np.mean(briers_mc)) if briers_mc else None
+ brier_top1_val = float(np.mean(briers_top1)) if briers_top1 else None
+
+ by_type_summary = {}
+ for qt, name in QTYPE_NAMES.items():
+ t_losses = by_type[qt]["losses"]
+ t_corrects = by_type[qt]["corrects"]
+ t_confs = by_type[qt]["confs"]
+ t_briers_mc = by_type[qt]["briers_mc"]
+ t_briers_top1 = by_type[qt]["briers_top1"]
+ if t_losses:
+ t_ece = float(ece_score(np.asarray(t_confs, dtype=float), np.asarray(t_corrects, dtype=bool))) if len(t_corrects) >= 2 else None
+ t_brier = float(np.mean(t_briers_mc))
+ t_brier_top1 = float(np.mean(t_briers_top1))
+ by_type_summary[name] = {
+ "items": len(t_losses),
+ "loss": round(float(np.mean(t_losses)), 4),
+ "accuracy": round(float(np.mean(t_corrects)), 4),
+ "mean_confidence": round(float(np.mean(t_confs)), 4),
+ "ece": round(t_ece, 4) if t_ece is not None else None,
+ "brier": round(t_brier, 4) if t_brier is not None else None,
+ "brier_top1": round(t_brier_top1, 4) if t_brier_top1 is not None else None,
+ }
+
+ return {
+ "items": total_items,
+ "loss": round(mean_loss, 4),
+ "accuracy": round(acc, 4),
+ "mean_confidence": round(mean_conf, 4),
+ "ece": round(ece_val, 4) if ece_val is not None else None,
+ "brier": round(brier_val, 4) if brier_val is not None else None,
+ "brier_top1": round(brier_top1_val, 4) if brier_top1_val is not None else None,
+ "by_type": by_type_summary,
+ }
+
+
+ def collapse_stats(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[Dict[str, float]]:
+ """Mean within-row logit range, mean soft-CE and mean ln K on records with k >= 2.
+
+ A 1-option row cannot collapse to a prior, so it is skipped. Empty input, or only
+ 1-option rows, returns None.
+ """
+ import numpy as np
+
+ ranges, ces, lnks = [], [], []
+ for _qt, logits, target, k in records:
+ k = int(k)
+ if k < 2:
+ continue
+ z = np.asarray(logits, dtype=float).ravel()[:k]
+ t = np.asarray(target, dtype=float).ravel()[:k]
+ mass = float(t.sum())
+ if mass <= 0.0:
+ continue
+ t = t / mass
+ ranges.append(float(z.max() - z.min()))
+ logp = z - np.logaddexp.reduce(z)
+ ces.append(float(-(t * logp).sum()))
+ lnks.append(math.log(k))
+ if not ranges:
+ return None
+ n = len(ranges)
+ return {
+ "n": float(n),
+ "mean_logit_range": sum(ranges) / n,
+ "mean_ce": sum(ces) / n,
+ "mean_ln_k": sum(lnks) / n,
+ }
+
+
+ def prior_collapse_message(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[str]:
+ """Warning text if calibration logits have collapsed to the class prior, else None.
+
+ The default 4-epoch budget is sized for a few thousand typed decisions. On a few
+ hundred or a thousand labelled rows the head can finish with every option at the
+ same logit -- chance-level, and silent. #963 measured a mean within-row range of
+ 0.01 against 0.62 for the untuned checkpoint, and training CE stuck at ln K.
+
+ Fired when the mean within-row logit range is below `COLLAPSE_LOGIT_RANGE`, or when
+ mean soft-CE on the same records is within `COLLAPSE_CE_REL` of mean ln K.
+ """
+ stats = collapse_stats(records)
+ if stats is None:
+ return None
+ reasons = []
+ if stats["mean_logit_range"] < COLLAPSE_LOGIT_RANGE:
+ reasons.append("mean within-row logit range %.3g" % stats["mean_logit_range"])
+ mean_ce, mean_lnk = stats["mean_ce"], stats["mean_ln_k"]
+ if mean_lnk > 0.0 and abs(mean_ce - mean_lnk) <= COLLAPSE_CE_REL * mean_lnk:
+ reasons.append("calibration cross-entropy %.3f is within %.0f%% of ln K ~ %.3f"
+ % (mean_ce, COLLAPSE_CE_REL * 100.0, mean_lnk))
+ if not reasons:
+ return None
+ return ("laya.train: predictions collapsed to the class prior (%s). "
+ "The default epoch budget is often too small on a few hundred or a thousand "
+ "labelled rows. Try more epochs, more data, or another seed."
+ % "; ".join(reasons))
+
+
def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,
device: Optional[str] = "auto") -> Dict[str, Any]:
"""Preprocess, train, calibrate and save; returns a summary of the run.