Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #968 from tiagovilasboas/fix/train-prior-collapse-warning
Nandakishore274b6ec8962
diff --cc laya/train.pyindex 7d85dfdf,6bf3b0fb..facf1cfb--- a/laya/train.py+++ b/laya/train.py@@@ -764,125 -763,69 +774,188 @@@ def calibration_report(records: Sequencreturn report+def evaluate_records(records: Sequence[Tuple[int, Any, Any, int]],+ temperature: Optional[Sequence[float]] = None,+ temperature_by_options: Optional[Dict[str, float]] = None) -> Dict[str, Any]:+ """Evaluate loss, accuracy, top-1 confidence, ECE and Brier scores on `records`.++ `records` is a sequence of `(qtype, logits, target, k)` tuples as returned by+ `calibration_records`. Predictions and confidences are scaled using `temperature` /+ `temperature_by_options` if provided. Returns overall metrics and per-question-type+ breakdowns.++ `brier` is the multi-class Brier score over the full probability vector:+ `mean(sum((p - target)**2))`. `brier_top1` is the binary Brier score of top-1+ confidence as P(correct): `mean((max(p) - correct)**2)`.+ """+ import numpy as np++ if not records:+ return {+ "items": 0,+ "loss": 0.0,+ "accuracy": 0.0,+ "mean_confidence": 0.0,+ "ece": None,+ "brier": None,+ "brier_top1": None,+ "by_type": {},+ }++ losses: List[float] = []+ corrects: List[bool] = []+ confs: List[float] = []+ briers_mc: List[float] = []+ briers_top1: List[float] = []+ by_type: Dict[int, Dict[str, List[Any]]] = {+ qt: {"losses": [], "corrects": [], "confs": [], "briers_mc": [], "briers_top1": []} for qt in QTYPE_NAMES+ }++ for qt, logits, target, k in records:+ z = np.asarray(logits[:k], dtype=float)+ t = np.asarray(target[:k], dtype=float)+ bucket = temp_bucket(qt, k)+ if temperature_by_options and bucket in temperature_by_options:+ scale = float(temperature_by_options[bucket])+ elif temperature is not None and isinstance(temperature, (list, tuple)) and qt < len(temperature):+ scale = float(temperature[qt])+ elif isinstance(temperature, (int, float)):+ scale = float(temperature)+ else:+ scale = 1.0+ scale = max(scale, 1e-4)+ z_scaled = z / scale+ z_max = np.max(z_scaled)+ exp_z = np.exp(z_scaled - z_max)+ sum_exp_z = np.sum(exp_z)+ p = exp_z / sum_exp_z+ log_p = z_scaled - z_max - np.log(sum_exp_z)+ loss = float(-np.sum(t * log_p))++ pred = int(np.argmax(p))+ gold = int(np.argmax(t))+ correct = bool(pred == gold)+ conf = float(np.max(p))+ b_mc = float(np.sum((p - t) ** 2))+ b_top1 = float((conf - float(correct)) ** 2)++ losses.append(loss)+ corrects.append(correct)+ confs.append(conf)+ briers_mc.append(b_mc)+ briers_top1.append(b_top1)++ if qt in by_type:+ by_type[qt]["losses"].append(loss)+ by_type[qt]["corrects"].append(correct)+ by_type[qt]["confs"].append(conf)+ by_type[qt]["briers_mc"].append(b_mc)+ by_type[qt]["briers_top1"].append(b_top1)++ total_items = len(records)+ acc = float(np.mean(corrects)) if corrects else 0.0+ mean_loss = float(np.mean(losses)) if losses else 0.0+ mean_conf = float(np.mean(confs)) if confs else 0.0+ ece_val = float(ece_score(np.asarray(confs, dtype=float), np.asarray(corrects, dtype=bool))) if len(corrects) >= 2 else None+ brier_val = float(np.mean(briers_mc)) if briers_mc else None+ brier_top1_val = float(np.mean(briers_top1)) if briers_top1 else None++ by_type_summary = {}+ for qt, name in QTYPE_NAMES.items():+ t_losses = by_type[qt]["losses"]+ t_corrects = by_type[qt]["corrects"]+ t_confs = by_type[qt]["confs"]+ t_briers_mc = by_type[qt]["briers_mc"]+ t_briers_top1 = by_type[qt]["briers_top1"]+ if t_losses:+ t_ece = float(ece_score(np.asarray(t_confs, dtype=float), np.asarray(t_corrects, dtype=bool))) if len(t_corrects) >= 2 else None+ t_brier = float(np.mean(t_briers_mc))+ t_brier_top1 = float(np.mean(t_briers_top1))+ by_type_summary[name] = {+ "items": len(t_losses),+ "loss": round(float(np.mean(t_losses)), 4),+ "accuracy": round(float(np.mean(t_corrects)), 4),+ "mean_confidence": round(float(np.mean(t_confs)), 4),+ "ece": round(t_ece, 4) if t_ece is not None else None,+ "brier": round(t_brier, 4) if t_brier is not None else None,+ "brier_top1": round(t_brier_top1, 4) if t_brier_top1 is not None else None,+ }++ return {+ "items": total_items,+ "loss": round(mean_loss, 4),+ "accuracy": round(acc, 4),+ "mean_confidence": round(mean_conf, 4),+ "ece": round(ece_val, 4) if ece_val is not None else None,+ "brier": round(brier_val, 4) if brier_val is not None else None,+ "brier_top1": round(brier_top1_val, 4) if brier_top1_val is not None else None,+ "by_type": by_type_summary,+ }+++ def collapse_stats(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[Dict[str, float]]:+ """Mean within-row logit range, mean soft-CE and mean ln K on records with k >= 2.++ A 1-option row cannot collapse to a prior, so it is skipped. Empty input, or only+ 1-option rows, returns None.+ """+ import numpy as np++ ranges, ces, lnks = [], [], []+ for _qt, logits, target, k in records:+ k = int(k)+ if k < 2:+ continue+ z = np.asarray(logits, dtype=float).ravel()[:k]+ t = np.asarray(target, dtype=float).ravel()[:k]+ mass = float(t.sum())+ if mass <= 0.0:+ continue+ t = t / mass+ ranges.append(float(z.max() - z.min()))+ logp = z - np.logaddexp.reduce(z)+ ces.append(float(-(t * logp).sum()))+ lnks.append(math.log(k))+ if not ranges:+ return None+ n = len(ranges)+ return {+ "n": float(n),+ "mean_logit_range": sum(ranges) / n,+ "mean_ce": sum(ces) / n,+ "mean_ln_k": sum(lnks) / n,+ }+++ def prior_collapse_message(records: Sequence[Tuple[int, Any, Any, int]]) -> Optional[str]:+ """Warning text if calibration logits have collapsed to the class prior, else None.++ The default 4-epoch budget is sized for a few thousand typed decisions. On a few+ hundred or a thousand labelled rows the head can finish with every option at the+ same logit -- chance-level, and silent. #963 measured a mean within-row range of+ 0.01 against 0.62 for the untuned checkpoint, and training CE stuck at ln K.++ Fired when the mean within-row logit range is below `COLLAPSE_LOGIT_RANGE`, or when+ mean soft-CE on the same records is within `COLLAPSE_CE_REL` of mean ln K.+ """+ stats = collapse_stats(records)+ if stats is None:+ return None+ reasons = []+ if stats["mean_logit_range"] < COLLAPSE_LOGIT_RANGE:+ reasons.append("mean within-row logit range %.3g" % stats["mean_logit_range"])+ mean_ce, mean_lnk = stats["mean_ce"], stats["mean_ln_k"]+ if mean_lnk > 0.0 and abs(mean_ce - mean_lnk) <= COLLAPSE_CE_REL * mean_lnk:+ reasons.append("calibration cross-entropy %.3f is within %.0f%% of ln K ~ %.3f"+ % (mean_ce, COLLAPSE_CE_REL * 100.0, mean_lnk))+ if not reasons:+ return None+ return ("laya.train: predictions collapsed to the class prior (%s). "+ "The default epoch budget is often too small on a few hundred or a thousand "+ "labelled rows. Try more epochs, more data, or another seed."+ % "; ".join(reasons))++def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,device: Optional[str] = "auto") -> Dict[str, Any]:"""Preprocess, train, calibrate and save; returns a summary of the run.