Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(train): add --eval, abstention threshold fitting, and before/after train_report.json (#887)
Swatantra Yadav212bdfa9fd8d
diff --git a/laya/train.py b/laya/train.pyindex cca4e6a0..7d85dfdf 100644--- a/laya/train.py+++ b/laya/train.py@@ -37,7 +37,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuplimport torch-from .calibrate import MIN_TYPE_N, fit_temperature_map+from .calibrate import MIN_TYPE_N, fit_abstention_thresholds, fit_temperature_mapfrom .common import (OPTION_LAYOUTS,QTYPE_NAMES,@@ -48,10 +48,12 @@ from .common import (build_model,build_sequence,collate_items,+ ece_score,encode_text,proper_reward,render_options,serialize_state,+ temp_bucket,uses_parallel_layout,)@@ -107,6 +109,9 @@ class TrainConfig:label_column: str = "label"question_id: str = "label"instructions: Optional[str] = None+ eval_data: Optional[str] = None+ target_error: float = 0.10+ min_abstain_n: int = 10def validate(self) -> None:if self.loss not in LOSSES:@@ -125,6 +130,12 @@ class TrainConfig:raise ValueError("calib_frac must be in [0, 1), got %r" % (self.calib_frac,))if isinstance(self.label_smoothing, bool) or not (0.0 <= self.label_smoothing < 1.0):raise ValueError("label_smoothing must be in [0, 1), got %r" % (self.label_smoothing,))+ if isinstance(self.target_error, bool) or not (0.0 <= self.target_error <= 1.0):+ raise ValueError("target_error must be in [0, 1], got %r" % (self.target_error,))+ if isinstance(self.min_abstain_n, bool) or not isinstance(self.min_abstain_n, int) or self.min_abstain_n < 1:+ raise ValueError("min_abstain_n must be a positive integer, got %r" % (self.min_abstain_n,))+ if self.eval_data is not None and not isinstance(self.eval_data, str):+ raise ValueError("eval_data must be a string path, got %r" % (self.eval_data,))# ------------------------------------------------------------------------------------- data@@ -753,6 +764,125 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],return report+def evaluate_records(records: Sequence[Tuple[int, Any, Any, int]],+ temperature: Optional[Sequence[float]] = None,+ temperature_by_options: Optional[Dict[str, float]] = None) -> Dict[str, Any]:+ """Evaluate loss, accuracy, top-1 confidence, ECE and Brier scores on `records`.++ `records` is a sequence of `(qtype, logits, target, k)` tuples as returned by+ `calibration_records`. Predictions and confidences are scaled using `temperature` /+ `temperature_by_options` if provided. Returns overall metrics and per-question-type+ breakdowns.++ `brier` is the multi-class Brier score over the full probability vector:+ `mean(sum((p - target)**2))`. `brier_top1` is the binary Brier score of top-1+ confidence as P(correct): `mean((max(p) - correct)**2)`.+ """+ import numpy as np++ if not records:+ return {+ "items": 0,+ "loss": 0.0,+ "accuracy": 0.0,+ "mean_confidence": 0.0,+ "ece": None,+ "brier": None,+ "brier_top1": None,+ "by_type": {},+ }++ losses: List[float] = []+ corrects: List[bool] = []+ confs: List[float] = []+ briers_mc: List[float] = []+ briers_top1: List[float] = []+ by_type: Dict[int, Dict[str, List[Any]]] = {+ qt: {"losses": [], "corrects": [], "confs": [], "briers_mc": [], "briers_top1": []} for qt in QTYPE_NAMES+ }++ for qt, logits, target, k in records:+ z = np.asarray(logits[:k], dtype=float)+ t = np.asarray(target[:k], dtype=float)+ bucket = temp_bucket(qt, k)+ if temperature_by_options and bucket in temperature_by_options:+ scale = float(temperature_by_options[bucket])+ elif temperature is not None and isinstance(temperature, (list, tuple)) and qt < len(temperature):+ scale = float(temperature[qt])+ elif isinstance(temperature, (int, float)):+ scale = float(temperature)+ else:+ scale = 1.0+ scale = max(scale, 1e-4)+ z_scaled = z / scale+ z_max = np.max(z_scaled)+ exp_z = np.exp(z_scaled - z_max)+ sum_exp_z = np.sum(exp_z)+ p = exp_z / sum_exp_z+ log_p = z_scaled - z_max - np.log(sum_exp_z)+ loss = float(-np.sum(t * log_p))++ pred = int(np.argmax(p))+ gold = int(np.argmax(t))+ correct = bool(pred == gold)+ conf = float(np.max(p))+ b_mc = float(np.sum((p - t) ** 2))+ b_top1 = float((conf - float(correct)) ** 2)++ losses.append(loss)+ corrects.append(correct)+ confs.append(conf)+ briers_mc.append(b_mc)+ briers_top1.append(b_top1)++ if qt in by_type:+ by_type[qt]["losses"].append(loss)+ by_type[qt]["corrects"].append(correct)+ by_type[qt]["confs"].append(conf)+ by_type[qt]["briers_mc"].append(b_mc)+ by_type[qt]["briers_top1"].append(b_top1)++ total_items = len(records)+ acc = float(np.mean(corrects)) if corrects else 0.0+ mean_loss = float(np.mean(losses)) if losses else 0.0+ mean_conf = float(np.mean(confs)) if confs else 0.0+ ece_val = float(ece_score(np.asarray(confs, dtype=float), np.asarray(corrects, dtype=bool))) if len(corrects) >= 2 else None+ brier_val = float(np.mean(briers_mc)) if briers_mc else None+ brier_top1_val = float(np.mean(briers_top1)) if briers_top1 else None++ by_type_summary = {}+ for qt, name in QTYPE_NAMES.items():+ t_losses = by_type[qt]["losses"]+ t_corrects = by_type[qt]["corrects"]+ t_confs = by_type[qt]["confs"]+ t_briers_mc = by_type[qt]["briers_mc"]+ t_briers_top1 = by_type[qt]["briers_top1"]+ if t_losses:+ t_ece = float(ece_score(np.asarray(t_confs, dtype=float), np.asarray(t_corrects, dtype=bool))) if len(t_corrects) >= 2 else None+ t_brier = float(np.mean(t_briers_mc))+ t_brier_top1 = float(np.mean(t_briers_top1))+ by_type_summary[name] = {+ "items": len(t_losses),+ "loss": round(float(np.mean(t_losses)), 4),+ "accuracy": round(float(np.mean(t_corrects)), 4),+ "mean_confidence": round(float(np.mean(t_confs)), 4),+ "ece": round(t_ece, 4) if t_ece is not None else None,+ "brier": round(t_brier, 4) if t_brier is not None else None,+ "brier_top1": round(t_brier_top1, 4) if t_brier_top1 is not None else None,+ }++ return {+ "items": total_items,+ "loss": round(mean_loss, 4),+ "accuracy": round(acc, 4),+ "mean_confidence": round(mean_conf, 4),+ "ece": round(ece_val, 4) if ece_val is not None else None,+ "brier": round(brier_val, 4) if brier_val is not None else None,+ "brier_top1": round(brier_top1_val, 4) if brier_top1_val is not None else None,+ "by_type": by_type_summary,+ }++def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,device: Optional[str] = "auto") -> Dict[str, Any]:"""Preprocess, train, calibrate and save; returns a summary of the run.@@ -779,8 +909,111 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCif not items:raise ValueError("%s produced no training items (skipped: %r)" % (data, skipped))train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)- print("train items %d, calibration items %d, skipped %r, device %s"- % (len(train_items), len(calib_items), skipped, dev), flush=True)++ eval_skipped = {}+ eval_mode = None+ is_held_out = False+ eval_source = None+ eval_items = []+ eval_note = None++ if config.eval_data:+ eval_source = config.eval_data+ eval_rows = read_data(config.eval_data, text_column=config.text_column, label_column=config.label_column,+ question_id=config.question_id, instructions=config.instructions)+ eval_items, eval_skipped = items_from_rows(tok, eval_rows, max_len, head_max_len, label_smoothing=0.0)++ if not eval_items:+ eval_mode = None+ is_held_out = False+ eval_note = ("Evaluation file %s yielded 0 usable items (skipped: %r); "+ "no evaluation performed." % (config.eval_data, eval_skipped))+ warnings.warn("laya.train: %s" % (eval_note,), RuntimeWarning, stacklevel=2)+ print("eval items 0 from %s (skipped %r)" % (config.eval_data, eval_skipped), flush=True)+ else:+ # Build input signatures from normalized, usable items:+ # (state token ids, normalized question schema).+ # Checked separately from targets to detect shared inputs even when labels differ (#967).+ train_input_sigs = {+ (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True))+ for it in train_items+ }+ calib_input_sigs = {+ (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True))+ for it in calib_items+ }++ train_overlap_count = sum(+ 1 for it in eval_items+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in train_input_sigs+ )+ calib_overlap_count = sum(+ 1 for it in eval_items+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in calib_input_sigs+ )+ total_overlap_count = sum(+ 1 for it in eval_items+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in train_input_sigs+ or (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in calib_input_sigs+ )++ if total_overlap_count > 0:+ eval_mode = "overlapping_eval"+ is_held_out = False+ if train_overlap_count > 0 and calib_overlap_count > 0:+ eval_note = (+ "Evaluation set contains inputs that overlap training data (%d) and calibration data (%d) "+ "(total %d/%d items); metrics do not reflect strictly independent held-out evaluation."+ % (train_overlap_count, calib_overlap_count, total_overlap_count, len(eval_items))+ )+ elif train_overlap_count > 0:+ eval_note = (+ "Evaluation set contains rows that overlap training data (%d/%d items); "+ "metrics do not reflect strictly independent held-out evaluation."+ % (train_overlap_count, len(eval_items))+ )+ else:+ eval_note = (+ "Evaluation set contains inputs that overlap calibration data (%d/%d items); "+ "metrics do not reflect strictly independent held-out evaluation."+ % (calib_overlap_count, len(eval_items))+ )+ print("eval items %d from %s (skipped %r, %d overlapping training/calibration items)"+ % (len(eval_items), config.eval_data, eval_skipped, total_overlap_count), flush=True)+ else:+ eval_mode = "held_out"+ is_held_out = True+ eval_note = "Metrics reflect independent held-out evaluation."+ print("eval items %d from %s (skipped %r, 0 overlapping training/calibration items)"+ % (len(eval_items), config.eval_data, eval_skipped), flush=True)+ elif calib_items:+ eval_items = calib_items+ eval_source = "calibration_slice"+ eval_mode = "in_sample_calibration"+ is_held_out = False+ eval_note = ("Metrics reflect in-sample calibration fit, not independent held-out evaluation. "+ "Provide --eval for held-out validation.")+ else:+ eval_items = []+ eval_source = None+ eval_mode = None+ is_held_out = False+ eval_note = "No evaluation performed (0 calibration items and no --eval set was provided)."+ warnings.warn("laya.train: dataset has 0 calibration items and no --eval set was provided; "+ "skipping evaluation to avoid reporting training fit as generalization.",+ RuntimeWarning, stacklevel=2)++ print("train items %d, calibration items %d, eval items %d%s, skipped %r, device %s"+ % (len(train_items), len(calib_items), len(eval_items),+ (" (" + str(eval_source) + ")") if eval_source else "", skipped, dev), flush=True)++ # Evaluate base checkpoint on eval items prior to training+ before_eval = None+ if eval_items:+ base_records = calibration_records(model, tok, eval_items, dev, max_len, head_max_len, parallel=parallel)+ base_temp = cfg.get("temperature")+ base_temp_by_options = cfg.get("temperature_by_options")+ before_eval = evaluate_records(base_records, base_temp, base_temp_by_options)def checkpoint_latest(epoch, _loss):save_checkpoint(model, tok, dict(cfg, max_len=max_len, head_max_len=head_max_len),@@ -794,19 +1027,103 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCcalibration = calibration_report(records, fitted["temperature"])for name, entry in calibration.items():for issue in entry["issues"]:- # Logged as a warning rather than left to the report alone: a checkpoint whose- # confidences were never calibrated looks the same as one that was.warnings.warn("laya.train: %s calibration: %s" % (name, issue), RuntimeWarning, stacklevel=2)++ try:+ abstention_thresholds = fit_abstention_thresholds(+ records,+ temperature=fitted["temperature"],+ temperature_by_options=fitted["temperature_by_options"],+ target_error=config.target_error,+ min_bucket_n=config.min_abstain_n,+ )+ except Exception as e:+ warnings.warn("laya.train: failed to fit abstention thresholds: %s" % (e,), RuntimeWarning, stacklevel=2)+ abstention_thresholds = {}++ # Evaluate fine-tuned checkpoint at fitted temperatures+ after_eval = None+ comparison = None+ if eval_items:+ after_records = calibration_records(model, tok, eval_items, dev, max_len, head_max_len, parallel=parallel)+ after_eval = evaluate_records(after_records, fitted["temperature"], fitted["temperature_by_options"])+ comparison = {+ "delta_accuracy": round(after_eval["accuracy"] - before_eval["accuracy"], 4),+ "delta_loss": round(after_eval["loss"] - before_eval["loss"], 4),+ "delta_ece": round(after_eval["ece"] - before_eval["ece"], 4) if (after_eval["ece"] is not None and before_eval["ece"] is not None) else None,+ "delta_brier": round(after_eval["brier"] - before_eval["brier"], 4) if (after_eval["brier"] is not None and before_eval["brier"] is not None) else None,+ "delta_brier_top1": round(after_eval["brier_top1"] - before_eval["brier_top1"], 4) if (after_eval["brier_top1"] is not None and before_eval["brier_top1"] is not None) else None,+ "delta_mean_confidence": round(after_eval["mean_confidence"] - before_eval["mean_confidence"], 4),+ }+ if eval_mode == "held_out":+ print("\n=== Evaluation (before vs after fine-tuning on %d held-out items from %s) ==="+ % (len(eval_items), eval_source), flush=True)+ elif eval_mode == "overlapping_eval":+ print("\n=== Evaluation (before vs after fine-tuning on %d items from %s; overlaps training data) ==="+ % (len(eval_items), eval_source), flush=True)+ else:+ print("\n=== Calibration Evidence (in-sample calibration slice, %d items; not held-out) ==="+ % (len(eval_items)), flush=True)+ print("%-22s %-14s %-14s %-14s" % ("Metric", "Before", "After", "Delta"), flush=True)+ print("-" * 65, flush=True)+ for metric, key in [("Accuracy", "accuracy"), ("Loss", "loss"), ("ECE", "ece"),+ ("Brier (multi-class)", "brier"), ("Brier (top-1)", "brier_top1"),+ ("Mean Confidence", "mean_confidence")]:+ b_val = before_eval.get(key)+ a_val = after_eval.get(key)+ d_val = comparison.get("delta_" + key)+ b_str = "%.4f" % b_val if b_val is not None else "N/A"+ a_str = "%.4f" % a_val if a_val is not None else "N/A"+ d_str = ("%+.4f" % d_val) if d_val is not None else "N/A"+ print("%-22s %-14s %-14s %-14s" % (metric, b_str, a_str, d_str), flush=True)+ print("-" * 65 + "\n", flush=True)++ if eval_note is None:+ if eval_mode == "held_out":+ eval_note = "Metrics reflect independent held-out evaluation."+ elif eval_mode == "overlapping_eval":+ eval_note = ("Evaluation set contains rows that overlap training data; "+ "metrics do not reflect strictly independent held-out evaluation.")+ elif eval_mode == "in_sample_calibration":+ eval_note = ("Metrics reflect in-sample calibration fit, not independent held-out evaluation. "+ "Provide --eval for held-out validation.")+ else:+ eval_note = "No evaluation performed (0 calibration items and no --eval set was provided)."++ train_report = {+ "eval_source": eval_source,+ "eval_mode": eval_mode,+ "is_held_out": is_held_out,+ "eval_items": len(eval_items),+ "note": eval_note,+ "before": before_eval,+ "after": after_eval,+ "comparison": comparison,+ "calibration": calibration,+ "abstention_thresholds": abstention_thresholds,+ "training": asdict(config),+ }+out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,temperature=fitted["temperature"])- # An inherited bucket map takes precedence at inference and would mask the new fit.out_cfg.pop("temperature_by_options", None)if fitted["temperature_by_options"]:out_cfg["temperature_by_options"] = fitted["temperature_by_options"]out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config),- laya_train_calibration=calibration)+ laya_train_calibration=calibration,+ train_report=train_report)+ if abstention_thresholds:+ out_cfg["training"]["abstention_thresholds"] = abstention_thresholdssave_checkpoint(model, tok, out_cfg, output_dir)+ checkpoint_latest_dir = os.path.join(output_dir, "checkpoint_latest")+ if os.path.isdir(checkpoint_latest_dir):+ # Synchronize final calibrated config so checkpoint_latest matches the final artifact+ tmp_cfg = os.path.join(checkpoint_latest_dir, "rl_agent_config.json.tmp")+ with open(tmp_cfg, "w", encoding="utf-8") as f:+ json.dump(out_cfg, f, indent=2)+ os.replace(tmp_cfg, os.path.join(checkpoint_latest_dir, "rl_agent_config.json"))+# Save the questions schema alongside the checkpoint for inference reusesample_questions = {}for r in rows:@@ -816,28 +1133,36 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainCif qid not in sample_questions:sample_questions[qid] = qdefelif sample_questions[qid] != qdef:- # `warnings` is imported at module scope. A function-local `import warnings`- # here would make the name local for the whole of `finetune`, and the- # calibration warnings above it would then raise UnboundLocalError.warnings.warn("Conflicting schema detected for question %r across rows; ""keeping first seen definition." % (qid,))if sample_questions:- for d in (output_dir, os.path.join(output_dir, "checkpoint_latest")):- os.makedirs(d, exist_ok=True)- tmp_path = os.path.join(d, "questions.json.tmp")- with open(tmp_path, "w", encoding="utf-8") as f:- json.dump(sample_questions, f, indent=2)- os.replace(tmp_path, os.path.join(d, "questions.json"))+ for d in (output_dir, checkpoint_latest_dir):+ if os.path.isdir(d):+ tmp_path = os.path.join(d, "questions.json.tmp")+ with open(tmp_path, "w", encoding="utf-8") as f:+ json.dump(sample_questions, f, indent=2)+ os.replace(tmp_path, os.path.join(d, "questions.json"))++ # Save train_report.json beside questions.json and checkpoint+ for d in (output_dir, checkpoint_latest_dir):+ if os.path.isdir(d):+ tmp_report = os.path.join(d, "train_report.json.tmp")+ with open(tmp_report, "w", encoding="utf-8") as f:+ json.dump(train_report, f, indent=2)+ os.replace(tmp_report, os.path.join(d, "train_report.json"))return {"train_items": len(train_items),"calibration_items": len(calib_items),+ "eval_items": len(eval_items),"skipped": skipped,"epoch_loss": history,"temperature": fitted["temperature"],"temperature_by_options": fitted["temperature_by_options"],"n_by_bucket": fitted["n_by_bucket"],"calibration": calibration,+ "abstention_thresholds": abstention_thresholds,+ "train_report": train_report,"output_dir": output_dir,}@@ -869,6 +1194,17 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->"max_len": max_len,"head_max_len": head_max_len,}+ if config.eval_data:+ eval_rows = read_data(config.eval_data, text_column=config.text_column, label_column=config.label_column,+ question_id=config.question_id, instructions=config.instructions)+ eval_items, eval_skipped = items_from_rows(tok, eval_rows, max_len, head_max_len, label_smoothing=0.0)+ summary["eval_data"] = config.eval_data+ summary["eval_rows_read"] = len(eval_rows)+ summary["eval_valid_items"] = len(eval_items)+ summary["eval_skipped"] = eval_skipped+ print("dry-run: eval %d rows read, %d valid items, skipped: %r"+ % (len(eval_rows), len(eval_items), eval_skipped), flush=True)+print("dry-run: %d rows read, %d valid items, skipped: %r"% (len(rows), len(items), skipped), flush=True)return summarydiff --git a/laya/train_cli.py b/laya/train_cli.pyindex c7bdcde5..527d0c43 100644--- a/laya/train_cli.py+++ b/laya/train_cli.py@@ -26,6 +26,13 @@ def build_parser() -> argparse.ArgumentParser:required=True,help="Path to training data file (.jsonl or .csv).",)+ parser.add_argument(+ "--eval",+ "--eval-data",+ dest="eval_data",+ default=None,+ help="Path to evaluation dataset (.jsonl or .csv) for before/after assessment.",+ )parser.add_argument("--out","--output-dir",@@ -117,6 +124,18 @@ def build_parser() -> argparse.ArgumentParser:action="store_true",help="Freeze encoder weights, training classification heads only.",)+ parser.add_argument(+ "--target-error",+ type=float,+ default=0.10,+ help="Target error rate for fitting abstention thresholds in [0, 1] (default: 0.10).",+ )+ parser.add_argument(+ "--min-abstain-n",+ type=int,+ default=10,+ help="Minimum items in a bucket required to fit an abstention threshold (default: 10).",+ )parser.add_argument("--device",default="auto",@@ -185,6 +204,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:label_column=args.label_column,question_id=args.question_id,instructions=args.instructions,+ eval_data=args.eval_data,+ target_error=args.target_error,+ min_abstain_n=args.min_abstain_n,)try:diff --git a/tests/test_train.py b/tests/test_train.pyindex bd2ee6a7..c1a761e9 100644--- a/tests/test_train.py+++ b/tests/test_train.py@@ -45,6 +45,7 @@ from laya.train import ( # noqa: E402draw_option_order,encode_item,encode_state,+ evaluate_records,finetune,items_from_rows,make_item,@@ -590,9 +591,13 @@ class EndToEndTests(unittest.TestCase):def test_laya_train_cli_parser_and_validation(self):parser = build_train_parser()- args = parser.parse_args(["--data", "train.csv", "--out", "./out", "--label-smoothing", "0.1",- "--shuffle-options", "--freeze-encoder"])+ args = parser.parse_args(["--data", "train.csv", "--eval", "eval.csv", "--target-error", "0.05",+ "--min-abstain-n", "5", "--out", "./out", "--label-smoothing", "0.1",+ "--shuffle-options", "--freeze-encoder"])self.assertEqual(args.data, "train.csv")+ self.assertEqual(args.eval_data, "eval.csv")+ self.assertEqual(args.target_error, 0.05)+ self.assertEqual(args.min_abstain_n, 5)self.assertEqual(args.output_dir, "./out")self.assertEqual(args.label_smoothing, 0.1)self.assertTrue(args.shuffle_options)@@ -607,6 +612,63 @@ class EndToEndTests(unittest.TestCase):self.assertIsNone(dry_args.output_dir)self.assertTrue(dry_args.dry_run)+ def test_train_config_eval_validation(self):+ cfg = TrainConfig(eval_data="val.csv", target_error=0.15, min_abstain_n=20)+ cfg.validate()++ with self.assertRaises(ValueError):+ TrainConfig(target_error=-0.1).validate()+ with self.assertRaises(ValueError):+ TrainConfig(target_error=1.5).validate()+ with self.assertRaises(ValueError):+ TrainConfig(target_error=True).validate()+ with self.assertRaises(ValueError):+ TrainConfig(min_abstain_n=0).validate()+ with self.assertRaises(ValueError):+ TrainConfig(min_abstain_n=-5).validate()+ with self.assertRaises(ValueError):+ TrainConfig(eval_data=123).validate()++ def test_evaluate_records(self):+ # Empty records returns default dict+ empty = evaluate_records([])+ self.assertEqual(empty["items"], 0)+ self.assertIsNone(empty["ece"])+ self.assertIsNone(empty["brier"])+ self.assertIsNone(empty["brier_top1"])++ # Distinct Brier values for known input:+ # 3 options, uniform prediction (logits [0, 0, 0] -> p = [1/3, 1/3, 1/3]), one-hot target [1, 0, 0].+ # Multiclass Brier = (1/3 - 1)^2 + (1/3 - 0)^2 + (1/3 - 0)^2 = 4/9 + 1/9 + 1/9 = 6/9 = 0.6667+ # Top-1 Confidence Brier = (1/3 - 1)^2 = 4/9 = 0.4444+ uniform_recs = [(0, [0.0, 0.0, 0.0], [1.0, 0.0, 0.0], 3)]+ uniform_metrics = evaluate_records(uniform_recs)+ self.assertEqual(uniform_metrics["brier"], 0.6667)+ self.assertEqual(uniform_metrics["brier_top1"], 0.4444)+ self.assertNotEqual(uniform_metrics["brier"], uniform_metrics["brier_top1"])++ # Synthetic records: 10 choice items, 10 score items+ # Choice: 3 options, logits favor option 0 strongly, gold is option 0+ choice_recs = [(0, [5.0, -2.0, -2.0], [1.0, 0.0, 0.0], 3) for _ in range(10)]+ # Score: 5 levels, logits favor level 2, gold is level 2+ score_recs = [(1, [-2.0, -2.0, 5.0, -2.0, -2.0], [0.0, 0.0, 1.0, 0.0, 0.0], 5) for _ in range(10)]+ metrics = evaluate_records(choice_recs + score_recs)+ self.assertEqual(metrics["items"], 20)+ self.assertEqual(metrics["accuracy"], 1.0)+ self.assertGreater(metrics["mean_confidence"], 0.9)+ self.assertIsNotNone(metrics["ece"])+ self.assertIsNotNone(metrics["brier"])+ self.assertIsNotNone(metrics["brier_top1"])+ self.assertIn("choice", metrics["by_type"])+ self.assertIn("score", metrics["by_type"])+ self.assertEqual(metrics["by_type"]["choice"]["items"], 10)+ self.assertEqual(metrics["by_type"]["score"]["items"], 10)+ self.assertIn("brier_top1", metrics["by_type"]["choice"])++ # Temperature scaling softens confidences+ warm = evaluate_records(choice_recs, temperature=[5.0, 5.0, 5.0])+ self.assertLess(warm["mean_confidence"], metrics["by_type"]["choice"]["mean_confidence"])+def test_laya_train_cli_runs_on_csv(self):csv_path = self.root / "tickets.csv"lines = ["text,department"]@@ -634,10 +696,32 @@ class EndToEndTests(unittest.TestCase):self.assertTrue((out_dir / "model.safetensors").exists())self.assertTrue((out_dir / "rl_agent_config.json").exists())self.assertTrue((out_dir / "questions.json").exists())+ self.assertTrue((out_dir / "train_report.json").exists())+ self.assertTrue((out_dir / "checkpoint_latest" / "train_report.json").exists())+ self.assertTrue((out_dir / "checkpoint_latest" / "rl_agent_config.json").exists())++ out_cfg = json.loads((out_dir / "rl_agent_config.json").read_text(encoding="utf-8"))+ latest_cfg = json.loads((out_dir / "checkpoint_latest" / "rl_agent_config.json").read_text(encoding="utf-8"))+ # Top-level min_confidence ghost field must not be written to checkpoint config+ self.assertNotIn("min_confidence", out_cfg)+ # checkpoint_latest must be synchronized with final calibrated out_cfg+ self.assertEqual(latest_cfg["temperature"], out_cfg["temperature"])+ self.assertEqual(latest_cfg["training"]["train_report"], out_cfg["training"]["train_report"])+saved_q = json.loads((out_dir / "questions.json").read_text(encoding="utf-8"))self.assertIn("label", saved_q)self.assertEqual(saved_q["label"]["criteria"], ["billing", "technical"])+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertIsNone(report["eval_source"])+ self.assertIsNone(report["eval_mode"])+ self.assertFalse(report["is_held_out"])+ self.assertEqual(report["eval_items"], 0)+ self.assertIn("No evaluation performed", report["note"])+ self.assertIsNone(report["before"])+ self.assertIsNone(report["after"])+ self.assertIsNone(report["comparison"])+agent = load(str(out_dir), device="cpu")ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify","criteria": ["billing", "technical"]}})@@ -653,6 +737,228 @@ class EndToEndTests(unittest.TestCase):])self.assertEqual(dry_code, 0)+ def test_laya_train_cli_with_eval_file(self):+ train_path = self.root / "train_eval_test.csv"+ eval_path = self.root / "eval_eval_test.csv"+ train_lines = ["text,department"]+ for s in BILLING_STATES:+ train_lines.append(f"{s},billing")+ for s in TECH_STATES:+ train_lines.append(f"{s},technical")+ train_path.write_text("\n".join(train_lines), encoding="utf-8")++ # Independent, non-overlapping evaluation items+ eval_lines = [+ "text,department",+ "please dispute my credit card charge,billing",+ "why was my card debited twice,billing",+ "the API server returns 502 bad gateway,technical",+ "the database socket connection timed out,technical",+ ]+ eval_path.write_text("\n".join(eval_lines), encoding="utf-8")++ out_dir = self.root / "out_with_eval"+ code = train_cli_main([+ "--data", str(train_path),+ "--eval", str(eval_path),+ "--base", str(self.root / "base"),+ "--out", str(out_dir),+ "--text-column", "text",+ "--label-column", "department",+ "--epochs", "1",+ "--micro-batch", "4",+ "--grad-accum", "1",+ "--loss", "soft-ce",+ "--target-error", "0.08",+ "--min-abstain-n", "2",+ "--device", "cpu",+ ])+ self.assertEqual(code, 0)+ self.assertTrue((out_dir / "train_report.json").exists())+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_source"], str(eval_path))+ self.assertEqual(report["eval_mode"], "held_out")+ self.assertTrue(report["is_held_out"])+ self.assertIn("independent held-out evaluation", report["note"])+ self.assertEqual(report["eval_items"], 4)+ self.assertIn("before", report)+ self.assertIn("after", report)+ self.assertIn("comparison", report)+ self.assertIn("brier", report["after"])+ self.assertIn("brier_top1", report["after"])+ self.assertIn("delta_brier", report["comparison"])+ self.assertIn("delta_brier_top1", report["comparison"])++ def test_laya_train_cli_with_overlapping_eval_file(self):+ train_path = self.root / "train_overlap.csv"+ eval_path = self.root / "eval_overlap.csv"+ train_lines = ["text,department"]+ eval_lines = ["text,department"]+ for s in BILLING_STATES:+ train_lines.append(f"{s},billing")+ eval_lines.append(f"{s},billing")+ for s in TECH_STATES:+ train_lines.append(f"{s},technical")+ eval_lines.append(f"{s},technical")+ train_path.write_text("\n".join(train_lines), encoding="utf-8")+ eval_path.write_text("\n".join(eval_lines), encoding="utf-8")++ out_dir = self.root / "out_overlap"+ code = train_cli_main([+ "--data", str(train_path),+ "--eval", str(eval_path),+ "--base", str(self.root / "base"),+ "--out", str(out_dir),+ "--text-column", "text",+ "--label-column", "department",+ "--epochs", "1",+ "--micro-batch", "4",+ "--grad-accum", "1",+ "--loss", "soft-ce",+ "--device", "cpu",+ ])+ self.assertEqual(code, 0)+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_source"], str(eval_path))+ self.assertEqual(report["eval_mode"], "overlapping_eval")+ self.assertFalse(report["is_held_out"])+ self.assertIn("overlap training data", report["note"])+ self.assertIn("brier", report["after"])+ self.assertIn("brier_top1", report["after"])+ self.assertIn("delta_brier", report["comparison"])+ self.assertIn("delta_brier_top1", report["comparison"])++ def test_laya_train_cli_with_empty_or_skipped_eval_file(self):+ train_path = self.root / "train_norm.csv"+ eval_path = self.root / "eval_empty.jsonl"+ train_lines = ["text,department"]+ for s in BILLING_STATES:+ train_lines.append(f"{s},billing")+ for s in TECH_STATES:+ train_lines.append(f"{s},technical")+ train_path.write_text("\n".join(train_lines), encoding="utf-8")+ eval_path.write_text("", encoding="utf-8")++ out_dir = self.root / "out_empty_eval"+ code = train_cli_main([+ "--data", str(train_path),+ "--eval", str(eval_path),+ "--base", str(self.root / "base"),+ "--out", str(out_dir),+ "--text-column", "text",+ "--label-column", "department",+ "--epochs", "1",+ "--micro-batch", "4",+ "--grad-accum", "1",+ "--loss", "soft-ce",+ "--device", "cpu",+ ])+ self.assertEqual(code, 0)+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_source"], str(eval_path))+ self.assertIsNone(report["eval_mode"])+ self.assertFalse(report["is_held_out"])+ self.assertEqual(report["eval_items"], 0)+ self.assertIsNone(report["before"])+ self.assertIsNone(report["after"])+ self.assertIn("0 usable items", report["note"])++ def test_overlap_detection_across_gold_and_expected_formats(self):+ # Training row uses 'gold', eval row uses 'expected'+ train_file = self.root / "train_gold.jsonl"+ eval_file = self.root / "eval_expected.jsonl"+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}+ train_row = {"state": "please refund payment", "questions": {"label": q_def}, "gold": {"label": {"probabilities": {"billing": 1.0}}}}+ eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}+ train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")+ eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")++ out_dir = self.root / "out_gold_expected_overlap"+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_mode"], "overlapping_eval")+ self.assertFalse(report["is_held_out"])+ self.assertIn("overlap training data", report["note"])++ def test_overlap_detection_with_different_targets(self):+ # Training and eval have identical input text and question, but different target label+ train_file = self.root / "train_diff_tgt.jsonl"+ eval_file = self.root / "eval_diff_tgt.jsonl"+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}+ train_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}+ eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "technical"}}+ train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")+ eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")++ out_dir = self.root / "out_diff_tgt_overlap"+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_mode"], "overlapping_eval")+ self.assertFalse(report["is_held_out"])+ self.assertIn("overlap training data", report["note"])++ def test_overlap_detection_with_calibration_slice(self):+ # Training set has multiple items split into train and calib.+ # Eval set only contains an item that ended up in the calibration slice, not train_items.+ train_file = self.root / "train_calib_split.jsonl"+ eval_file = self.root / "eval_calib_split.jsonl"+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}++ states = ["refund please", "charged twice", "crash app", "screen error"]+ rows = [+ {"state": s, "questions": {"label": q_def}, "expected": {"label": "billing" if "refund" in s or "charged" in s else "technical"}}+ for s in states+ ]+ train_file.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")++ import random+ order = list(range(len(rows)))+ random.Random(42).shuffle(order)+ calib_row = rows[order[0]] # in calib set, not in train set+ eval_file.write_text(json.dumps(calib_row) + "\n", encoding="utf-8")++ out_dir = self.root / "out_calib_overlap"+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, calib_frac=0.5, calib_max=2, calib_seed=42, eval_data=str(eval_file))+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_mode"], "overlapping_eval")+ self.assertFalse(report["is_held_out"])+ self.assertIn("overlap calibration data", report["note"])++ def test_laya_train_in_sample_calibration_mode(self):+ csv_path = self.root / "calib_tickets.csv"+ lines = ["text,department"]+ for s in BILLING_STATES:+ lines.append(f"{s},billing")+ for s in TECH_STATES:+ lines.append(f"{s},technical")+ csv_path.write_text("\n".join(lines), encoding="utf-8")++ out_dir = self.root / "out_in_sample"+ cfg = TrainConfig(+ text_column="text",+ label_column="department",+ epochs=1,+ micro_batch=4,+ grad_accum=1,+ calib_frac=0.2,+ calib_max=10,+ target_error=0.10,+ min_abstain_n=2,+ )+ finetune(str(csv_path), str(self.root / "base"), str(out_dir), cfg, device="cpu")+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))+ self.assertEqual(report["eval_source"], "calibration_slice")+ self.assertEqual(report["eval_mode"], "in_sample_calibration")+ self.assertFalse(report["is_held_out"])+ self.assertIn("in-sample calibration fit", report["note"])+ self.assertGreater(report["eval_items"], 0)+ self.assertIn("before", report)+ self.assertIn("after", report)+ self.assertIn("comparison", report)+class CalibrationReportTests(unittest.TestCase):@staticmethod