Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(train): add --eval, abstention threshold fitting, and before/after train_report.json (#887)

Swatantra Yadav212bdfa9fd8d
diff --git a/laya/train.py b/laya/train.py
index cca4e6a0..7d85dfdf 100644
--- a/laya/train.py
+++ b/laya/train.py
@@ -37,7 +37,7 @@ from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tupl
import torch
-from .calibrate import MIN_TYPE_N, fit_temperature_map
+from .calibrate import MIN_TYPE_N, fit_abstention_thresholds, fit_temperature_map
from .common import (
OPTION_LAYOUTS,
QTYPE_NAMES,
@@ -48,10 +48,12 @@ from .common import (
build_model,
build_sequence,
collate_items,
+ ece_score,
encode_text,
proper_reward,
render_options,
serialize_state,
+ temp_bucket,
uses_parallel_layout,
)
@@ -107,6 +109,9 @@ class TrainConfig:
label_column: str = "label"
question_id: str = "label"
instructions: Optional[str] = None
+ eval_data: Optional[str] = None
+ target_error: float = 0.10
+ min_abstain_n: int = 10
def validate(self) -> None:
if self.loss not in LOSSES:
@@ -125,6 +130,12 @@ class TrainConfig:
raise ValueError("calib_frac must be in [0, 1), got %r" % (self.calib_frac,))
if isinstance(self.label_smoothing, bool) or not (0.0 <= self.label_smoothing < 1.0):
raise ValueError("label_smoothing must be in [0, 1), got %r" % (self.label_smoothing,))
+ if isinstance(self.target_error, bool) or not (0.0 <= self.target_error <= 1.0):
+ raise ValueError("target_error must be in [0, 1], got %r" % (self.target_error,))
+ if isinstance(self.min_abstain_n, bool) or not isinstance(self.min_abstain_n, int) or self.min_abstain_n < 1:
+ raise ValueError("min_abstain_n must be a positive integer, got %r" % (self.min_abstain_n,))
+ if self.eval_data is not None and not isinstance(self.eval_data, str):
+ raise ValueError("eval_data must be a string path, got %r" % (self.eval_data,))
# ------------------------------------------------------------------------------------- data
@@ -753,6 +764,125 @@ def calibration_report(records: Sequence[Tuple[int, Any, Any, int]],
return report
+def evaluate_records(records: Sequence[Tuple[int, Any, Any, int]],
+ temperature: Optional[Sequence[float]] = None,
+ temperature_by_options: Optional[Dict[str, float]] = None) -> Dict[str, Any]:
+ """Evaluate loss, accuracy, top-1 confidence, ECE and Brier scores on `records`.
+
+ `records` is a sequence of `(qtype, logits, target, k)` tuples as returned by
+ `calibration_records`. Predictions and confidences are scaled using `temperature` /
+ `temperature_by_options` if provided. Returns overall metrics and per-question-type
+ breakdowns.
+
+ `brier` is the multi-class Brier score over the full probability vector:
+ `mean(sum((p - target)**2))`. `brier_top1` is the binary Brier score of top-1
+ confidence as P(correct): `mean((max(p) - correct)**2)`.
+ """
+ import numpy as np
+
+ if not records:
+ return {
+ "items": 0,
+ "loss": 0.0,
+ "accuracy": 0.0,
+ "mean_confidence": 0.0,
+ "ece": None,
+ "brier": None,
+ "brier_top1": None,
+ "by_type": {},
+ }
+
+ losses: List[float] = []
+ corrects: List[bool] = []
+ confs: List[float] = []
+ briers_mc: List[float] = []
+ briers_top1: List[float] = []
+ by_type: Dict[int, Dict[str, List[Any]]] = {
+ qt: {"losses": [], "corrects": [], "confs": [], "briers_mc": [], "briers_top1": []} for qt in QTYPE_NAMES
+ }
+
+ for qt, logits, target, k in records:
+ z = np.asarray(logits[:k], dtype=float)
+ t = np.asarray(target[:k], dtype=float)
+ bucket = temp_bucket(qt, k)
+ if temperature_by_options and bucket in temperature_by_options:
+ scale = float(temperature_by_options[bucket])
+ elif temperature is not None and isinstance(temperature, (list, tuple)) and qt < len(temperature):
+ scale = float(temperature[qt])
+ elif isinstance(temperature, (int, float)):
+ scale = float(temperature)
+ else:
+ scale = 1.0
+ scale = max(scale, 1e-4)
+ z_scaled = z / scale
+ z_max = np.max(z_scaled)
+ exp_z = np.exp(z_scaled - z_max)
+ sum_exp_z = np.sum(exp_z)
+ p = exp_z / sum_exp_z
+ log_p = z_scaled - z_max - np.log(sum_exp_z)
+ loss = float(-np.sum(t * log_p))
+
+ pred = int(np.argmax(p))
+ gold = int(np.argmax(t))
+ correct = bool(pred == gold)
+ conf = float(np.max(p))
+ b_mc = float(np.sum((p - t) ** 2))
+ b_top1 = float((conf - float(correct)) ** 2)
+
+ losses.append(loss)
+ corrects.append(correct)
+ confs.append(conf)
+ briers_mc.append(b_mc)
+ briers_top1.append(b_top1)
+
+ if qt in by_type:
+ by_type[qt]["losses"].append(loss)
+ by_type[qt]["corrects"].append(correct)
+ by_type[qt]["confs"].append(conf)
+ by_type[qt]["briers_mc"].append(b_mc)
+ by_type[qt]["briers_top1"].append(b_top1)
+
+ total_items = len(records)
+ acc = float(np.mean(corrects)) if corrects else 0.0
+ mean_loss = float(np.mean(losses)) if losses else 0.0
+ mean_conf = float(np.mean(confs)) if confs else 0.0
+ ece_val = float(ece_score(np.asarray(confs, dtype=float), np.asarray(corrects, dtype=bool))) if len(corrects) >= 2 else None
+ brier_val = float(np.mean(briers_mc)) if briers_mc else None
+ brier_top1_val = float(np.mean(briers_top1)) if briers_top1 else None
+
+ by_type_summary = {}
+ for qt, name in QTYPE_NAMES.items():
+ t_losses = by_type[qt]["losses"]
+ t_corrects = by_type[qt]["corrects"]
+ t_confs = by_type[qt]["confs"]
+ t_briers_mc = by_type[qt]["briers_mc"]
+ t_briers_top1 = by_type[qt]["briers_top1"]
+ if t_losses:
+ t_ece = float(ece_score(np.asarray(t_confs, dtype=float), np.asarray(t_corrects, dtype=bool))) if len(t_corrects) >= 2 else None
+ t_brier = float(np.mean(t_briers_mc))
+ t_brier_top1 = float(np.mean(t_briers_top1))
+ by_type_summary[name] = {
+ "items": len(t_losses),
+ "loss": round(float(np.mean(t_losses)), 4),
+ "accuracy": round(float(np.mean(t_corrects)), 4),
+ "mean_confidence": round(float(np.mean(t_confs)), 4),
+ "ece": round(t_ece, 4) if t_ece is not None else None,
+ "brier": round(t_brier, 4) if t_brier is not None else None,
+ "brier_top1": round(t_brier_top1, 4) if t_brier_top1 is not None else None,
+ }
+
+ return {
+ "items": total_items,
+ "loss": round(mean_loss, 4),
+ "accuracy": round(acc, 4),
+ "mean_confidence": round(mean_conf, 4),
+ "ece": round(ece_val, 4) if ece_val is not None else None,
+ "brier": round(brier_val, 4) if brier_val is not None else None,
+ "brier_top1": round(brier_top1_val, 4) if brier_top1_val is not None else None,
+ "by_type": by_type_summary,
+ }
+
+
def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,
device: Optional[str] = "auto") -> Dict[str, Any]:
"""Preprocess, train, calibrate and save; returns a summary of the run.
@@ -779,8 +909,111 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
if not items:
raise ValueError("%s produced no training items (skipped: %r)" % (data, skipped))
train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)
- print("train items %d, calibration items %d, skipped %r, device %s"
- % (len(train_items), len(calib_items), skipped, dev), flush=True)
+
+ eval_skipped = {}
+ eval_mode = None
+ is_held_out = False
+ eval_source = None
+ eval_items = []
+ eval_note = None
+
+ if config.eval_data:
+ eval_source = config.eval_data
+ eval_rows = read_data(config.eval_data, text_column=config.text_column, label_column=config.label_column,
+ question_id=config.question_id, instructions=config.instructions)
+ eval_items, eval_skipped = items_from_rows(tok, eval_rows, max_len, head_max_len, label_smoothing=0.0)
+
+ if not eval_items:
+ eval_mode = None
+ is_held_out = False
+ eval_note = ("Evaluation file %s yielded 0 usable items (skipped: %r); "
+ "no evaluation performed." % (config.eval_data, eval_skipped))
+ warnings.warn("laya.train: %s" % (eval_note,), RuntimeWarning, stacklevel=2)
+ print("eval items 0 from %s (skipped %r)" % (config.eval_data, eval_skipped), flush=True)
+ else:
+ # Build input signatures from normalized, usable items:
+ # (state token ids, normalized question schema).
+ # Checked separately from targets to detect shared inputs even when labels differ (#967).
+ train_input_sigs = {
+ (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True))
+ for it in train_items
+ }
+ calib_input_sigs = {
+ (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True))
+ for it in calib_items
+ }
+
+ train_overlap_count = sum(
+ 1 for it in eval_items
+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in train_input_sigs
+ )
+ calib_overlap_count = sum(
+ 1 for it in eval_items
+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in calib_input_sigs
+ )
+ total_overlap_count = sum(
+ 1 for it in eval_items
+ if (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in train_input_sigs
+ or (tuple(it["state_ids"]), json.dumps(it["q"], sort_keys=True)) in calib_input_sigs
+ )
+
+ if total_overlap_count > 0:
+ eval_mode = "overlapping_eval"
+ is_held_out = False
+ if train_overlap_count > 0 and calib_overlap_count > 0:
+ eval_note = (
+ "Evaluation set contains inputs that overlap training data (%d) and calibration data (%d) "
+ "(total %d/%d items); metrics do not reflect strictly independent held-out evaluation."
+ % (train_overlap_count, calib_overlap_count, total_overlap_count, len(eval_items))
+ )
+ elif train_overlap_count > 0:
+ eval_note = (
+ "Evaluation set contains rows that overlap training data (%d/%d items); "
+ "metrics do not reflect strictly independent held-out evaluation."
+ % (train_overlap_count, len(eval_items))
+ )
+ else:
+ eval_note = (
+ "Evaluation set contains inputs that overlap calibration data (%d/%d items); "
+ "metrics do not reflect strictly independent held-out evaluation."
+ % (calib_overlap_count, len(eval_items))
+ )
+ print("eval items %d from %s (skipped %r, %d overlapping training/calibration items)"
+ % (len(eval_items), config.eval_data, eval_skipped, total_overlap_count), flush=True)
+ else:
+ eval_mode = "held_out"
+ is_held_out = True
+ eval_note = "Metrics reflect independent held-out evaluation."
+ print("eval items %d from %s (skipped %r, 0 overlapping training/calibration items)"
+ % (len(eval_items), config.eval_data, eval_skipped), flush=True)
+ elif calib_items:
+ eval_items = calib_items
+ eval_source = "calibration_slice"
+ eval_mode = "in_sample_calibration"
+ is_held_out = False
+ eval_note = ("Metrics reflect in-sample calibration fit, not independent held-out evaluation. "
+ "Provide --eval for held-out validation.")
+ else:
+ eval_items = []
+ eval_source = None
+ eval_mode = None
+ is_held_out = False
+ eval_note = "No evaluation performed (0 calibration items and no --eval set was provided)."
+ warnings.warn("laya.train: dataset has 0 calibration items and no --eval set was provided; "
+ "skipping evaluation to avoid reporting training fit as generalization.",
+ RuntimeWarning, stacklevel=2)
+
+ print("train items %d, calibration items %d, eval items %d%s, skipped %r, device %s"
+ % (len(train_items), len(calib_items), len(eval_items),
+ (" (" + str(eval_source) + ")") if eval_source else "", skipped, dev), flush=True)
+
+ # Evaluate base checkpoint on eval items prior to training
+ before_eval = None
+ if eval_items:
+ base_records = calibration_records(model, tok, eval_items, dev, max_len, head_max_len, parallel=parallel)
+ base_temp = cfg.get("temperature")
+ base_temp_by_options = cfg.get("temperature_by_options")
+ before_eval = evaluate_records(base_records, base_temp, base_temp_by_options)
def checkpoint_latest(epoch, _loss):
save_checkpoint(model, tok, dict(cfg, max_len=max_len, head_max_len=head_max_len),
@@ -794,19 +1027,103 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
calibration = calibration_report(records, fitted["temperature"])
for name, entry in calibration.items():
for issue in entry["issues"]:
- # Logged as a warning rather than left to the report alone: a checkpoint whose
- # confidences were never calibrated looks the same as one that was.
warnings.warn("laya.train: %s calibration: %s" % (name, issue), RuntimeWarning, stacklevel=2)
+
+ try:
+ abstention_thresholds = fit_abstention_thresholds(
+ records,
+ temperature=fitted["temperature"],
+ temperature_by_options=fitted["temperature_by_options"],
+ target_error=config.target_error,
+ min_bucket_n=config.min_abstain_n,
+ )
+ except Exception as e:
+ warnings.warn("laya.train: failed to fit abstention thresholds: %s" % (e,), RuntimeWarning, stacklevel=2)
+ abstention_thresholds = {}
+
+ # Evaluate fine-tuned checkpoint at fitted temperatures
+ after_eval = None
+ comparison = None
+ if eval_items:
+ after_records = calibration_records(model, tok, eval_items, dev, max_len, head_max_len, parallel=parallel)
+ after_eval = evaluate_records(after_records, fitted["temperature"], fitted["temperature_by_options"])
+ comparison = {
+ "delta_accuracy": round(after_eval["accuracy"] - before_eval["accuracy"], 4),
+ "delta_loss": round(after_eval["loss"] - before_eval["loss"], 4),
+ "delta_ece": round(after_eval["ece"] - before_eval["ece"], 4) if (after_eval["ece"] is not None and before_eval["ece"] is not None) else None,
+ "delta_brier": round(after_eval["brier"] - before_eval["brier"], 4) if (after_eval["brier"] is not None and before_eval["brier"] is not None) else None,
+ "delta_brier_top1": round(after_eval["brier_top1"] - before_eval["brier_top1"], 4) if (after_eval["brier_top1"] is not None and before_eval["brier_top1"] is not None) else None,
+ "delta_mean_confidence": round(after_eval["mean_confidence"] - before_eval["mean_confidence"], 4),
+ }
+ if eval_mode == "held_out":
+ print("\n=== Evaluation (before vs after fine-tuning on %d held-out items from %s) ==="
+ % (len(eval_items), eval_source), flush=True)
+ elif eval_mode == "overlapping_eval":
+ print("\n=== Evaluation (before vs after fine-tuning on %d items from %s; overlaps training data) ==="
+ % (len(eval_items), eval_source), flush=True)
+ else:
+ print("\n=== Calibration Evidence (in-sample calibration slice, %d items; not held-out) ==="
+ % (len(eval_items)), flush=True)
+ print("%-22s %-14s %-14s %-14s" % ("Metric", "Before", "After", "Delta"), flush=True)
+ print("-" * 65, flush=True)
+ for metric, key in [("Accuracy", "accuracy"), ("Loss", "loss"), ("ECE", "ece"),
+ ("Brier (multi-class)", "brier"), ("Brier (top-1)", "brier_top1"),
+ ("Mean Confidence", "mean_confidence")]:
+ b_val = before_eval.get(key)
+ a_val = after_eval.get(key)
+ d_val = comparison.get("delta_" + key)
+ b_str = "%.4f" % b_val if b_val is not None else "N/A"
+ a_str = "%.4f" % a_val if a_val is not None else "N/A"
+ d_str = ("%+.4f" % d_val) if d_val is not None else "N/A"
+ print("%-22s %-14s %-14s %-14s" % (metric, b_str, a_str, d_str), flush=True)
+ print("-" * 65 + "\n", flush=True)
+
+ if eval_note is None:
+ if eval_mode == "held_out":
+ eval_note = "Metrics reflect independent held-out evaluation."
+ elif eval_mode == "overlapping_eval":
+ eval_note = ("Evaluation set contains rows that overlap training data; "
+ "metrics do not reflect strictly independent held-out evaluation.")
+ elif eval_mode == "in_sample_calibration":
+ eval_note = ("Metrics reflect in-sample calibration fit, not independent held-out evaluation. "
+ "Provide --eval for held-out validation.")
+ else:
+ eval_note = "No evaluation performed (0 calibration items and no --eval set was provided)."
+
+ train_report = {
+ "eval_source": eval_source,
+ "eval_mode": eval_mode,
+ "is_held_out": is_held_out,
+ "eval_items": len(eval_items),
+ "note": eval_note,
+ "before": before_eval,
+ "after": after_eval,
+ "comparison": comparison,
+ "calibration": calibration,
+ "abstention_thresholds": abstention_thresholds,
+ "training": asdict(config),
+ }
+
out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,
temperature=fitted["temperature"])
- # An inherited bucket map takes precedence at inference and would mask the new fit.
out_cfg.pop("temperature_by_options", None)
if fitted["temperature_by_options"]:
out_cfg["temperature_by_options"] = fitted["temperature_by_options"]
out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config),
- laya_train_calibration=calibration)
+ laya_train_calibration=calibration,
+ train_report=train_report)
+ if abstention_thresholds:
+ out_cfg["training"]["abstention_thresholds"] = abstention_thresholds
save_checkpoint(model, tok, out_cfg, output_dir)
+ checkpoint_latest_dir = os.path.join(output_dir, "checkpoint_latest")
+ if os.path.isdir(checkpoint_latest_dir):
+ # Synchronize final calibrated config so checkpoint_latest matches the final artifact
+ tmp_cfg = os.path.join(checkpoint_latest_dir, "rl_agent_config.json.tmp")
+ with open(tmp_cfg, "w", encoding="utf-8") as f:
+ json.dump(out_cfg, f, indent=2)
+ os.replace(tmp_cfg, os.path.join(checkpoint_latest_dir, "rl_agent_config.json"))
+
# Save the questions schema alongside the checkpoint for inference reuse
sample_questions = {}
for r in rows:
@@ -816,28 +1133,36 @@ def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainC
if qid not in sample_questions:
sample_questions[qid] = qdef
elif sample_questions[qid] != qdef:
- # `warnings` is imported at module scope. A function-local `import warnings`
- # here would make the name local for the whole of `finetune`, and the
- # calibration warnings above it would then raise UnboundLocalError.
warnings.warn("Conflicting schema detected for question %r across rows; "
"keeping first seen definition." % (qid,))
if sample_questions:
- for d in (output_dir, os.path.join(output_dir, "checkpoint_latest")):
- os.makedirs(d, exist_ok=True)
- tmp_path = os.path.join(d, "questions.json.tmp")
- with open(tmp_path, "w", encoding="utf-8") as f:
- json.dump(sample_questions, f, indent=2)
- os.replace(tmp_path, os.path.join(d, "questions.json"))
+ for d in (output_dir, checkpoint_latest_dir):
+ if os.path.isdir(d):
+ tmp_path = os.path.join(d, "questions.json.tmp")
+ with open(tmp_path, "w", encoding="utf-8") as f:
+ json.dump(sample_questions, f, indent=2)
+ os.replace(tmp_path, os.path.join(d, "questions.json"))
+
+ # Save train_report.json beside questions.json and checkpoint
+ for d in (output_dir, checkpoint_latest_dir):
+ if os.path.isdir(d):
+ tmp_report = os.path.join(d, "train_report.json.tmp")
+ with open(tmp_report, "w", encoding="utf-8") as f:
+ json.dump(train_report, f, indent=2)
+ os.replace(tmp_report, os.path.join(d, "train_report.json"))
return {
"train_items": len(train_items),
"calibration_items": len(calib_items),
+ "eval_items": len(eval_items),
"skipped": skipped,
"epoch_loss": history,
"temperature": fitted["temperature"],
"temperature_by_options": fitted["temperature_by_options"],
"n_by_bucket": fitted["n_by_bucket"],
"calibration": calibration,
+ "abstention_thresholds": abstention_thresholds,
+ "train_report": train_report,
"output_dir": output_dir,
}
@@ -869,6 +1194,17 @@ def dry_run(data: str, model_dir: str, config: Optional[TrainConfig] = None) ->
"max_len": max_len,
"head_max_len": head_max_len,
}
+ if config.eval_data:
+ eval_rows = read_data(config.eval_data, text_column=config.text_column, label_column=config.label_column,
+ question_id=config.question_id, instructions=config.instructions)
+ eval_items, eval_skipped = items_from_rows(tok, eval_rows, max_len, head_max_len, label_smoothing=0.0)
+ summary["eval_data"] = config.eval_data
+ summary["eval_rows_read"] = len(eval_rows)
+ summary["eval_valid_items"] = len(eval_items)
+ summary["eval_skipped"] = eval_skipped
+ print("dry-run: eval %d rows read, %d valid items, skipped: %r"
+ % (len(eval_rows), len(eval_items), eval_skipped), flush=True)
+
print("dry-run: %d rows read, %d valid items, skipped: %r"
% (len(rows), len(items), skipped), flush=True)
return summary
diff --git a/laya/train_cli.py b/laya/train_cli.py
index c7bdcde5..527d0c43 100644
--- a/laya/train_cli.py
+++ b/laya/train_cli.py
@@ -26,6 +26,13 @@ def build_parser() -> argparse.ArgumentParser:
required=True,
help="Path to training data file (.jsonl or .csv).",
)
+ parser.add_argument(
+ "--eval",
+ "--eval-data",
+ dest="eval_data",
+ default=None,
+ help="Path to evaluation dataset (.jsonl or .csv) for before/after assessment.",
+ )
parser.add_argument(
"--out",
"--output-dir",
@@ -117,6 +124,18 @@ def build_parser() -> argparse.ArgumentParser:
action="store_true",
help="Freeze encoder weights, training classification heads only.",
)
+ parser.add_argument(
+ "--target-error",
+ type=float,
+ default=0.10,
+ help="Target error rate for fitting abstention thresholds in [0, 1] (default: 0.10).",
+ )
+ parser.add_argument(
+ "--min-abstain-n",
+ type=int,
+ default=10,
+ help="Minimum items in a bucket required to fit an abstention threshold (default: 10).",
+ )
parser.add_argument(
"--device",
default="auto",
@@ -185,6 +204,9 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
label_column=args.label_column,
question_id=args.question_id,
instructions=args.instructions,
+ eval_data=args.eval_data,
+ target_error=args.target_error,
+ min_abstain_n=args.min_abstain_n,
)
try:
diff --git a/tests/test_train.py b/tests/test_train.py
index bd2ee6a7..c1a761e9 100644
--- a/tests/test_train.py
+++ b/tests/test_train.py
@@ -45,6 +45,7 @@ from laya.train import ( # noqa: E402
draw_option_order,
encode_item,
encode_state,
+ evaluate_records,
finetune,
items_from_rows,
make_item,
@@ -590,9 +591,13 @@ class EndToEndTests(unittest.TestCase):
def test_laya_train_cli_parser_and_validation(self):
parser = build_train_parser()
- args = parser.parse_args(["--data", "train.csv", "--out", "./out", "--label-smoothing", "0.1",
- "--shuffle-options", "--freeze-encoder"])
+ args = parser.parse_args(["--data", "train.csv", "--eval", "eval.csv", "--target-error", "0.05",
+ "--min-abstain-n", "5", "--out", "./out", "--label-smoothing", "0.1",
+ "--shuffle-options", "--freeze-encoder"])
self.assertEqual(args.data, "train.csv")
+ self.assertEqual(args.eval_data, "eval.csv")
+ self.assertEqual(args.target_error, 0.05)
+ self.assertEqual(args.min_abstain_n, 5)
self.assertEqual(args.output_dir, "./out")
self.assertEqual(args.label_smoothing, 0.1)
self.assertTrue(args.shuffle_options)
@@ -607,6 +612,63 @@ class EndToEndTests(unittest.TestCase):
self.assertIsNone(dry_args.output_dir)
self.assertTrue(dry_args.dry_run)
+ def test_train_config_eval_validation(self):
+ cfg = TrainConfig(eval_data="val.csv", target_error=0.15, min_abstain_n=20)
+ cfg.validate()
+
+ with self.assertRaises(ValueError):
+ TrainConfig(target_error=-0.1).validate()
+ with self.assertRaises(ValueError):
+ TrainConfig(target_error=1.5).validate()
+ with self.assertRaises(ValueError):
+ TrainConfig(target_error=True).validate()
+ with self.assertRaises(ValueError):
+ TrainConfig(min_abstain_n=0).validate()
+ with self.assertRaises(ValueError):
+ TrainConfig(min_abstain_n=-5).validate()
+ with self.assertRaises(ValueError):
+ TrainConfig(eval_data=123).validate()
+
+ def test_evaluate_records(self):
+ # Empty records returns default dict
+ empty = evaluate_records([])
+ self.assertEqual(empty["items"], 0)
+ self.assertIsNone(empty["ece"])
+ self.assertIsNone(empty["brier"])
+ self.assertIsNone(empty["brier_top1"])
+
+ # Distinct Brier values for known input:
+ # 3 options, uniform prediction (logits [0, 0, 0] -> p = [1/3, 1/3, 1/3]), one-hot target [1, 0, 0].
+ # Multiclass Brier = (1/3 - 1)^2 + (1/3 - 0)^2 + (1/3 - 0)^2 = 4/9 + 1/9 + 1/9 = 6/9 = 0.6667
+ # Top-1 Confidence Brier = (1/3 - 1)^2 = 4/9 = 0.4444
+ uniform_recs = [(0, [0.0, 0.0, 0.0], [1.0, 0.0, 0.0], 3)]
+ uniform_metrics = evaluate_records(uniform_recs)
+ self.assertEqual(uniform_metrics["brier"], 0.6667)
+ self.assertEqual(uniform_metrics["brier_top1"], 0.4444)
+ self.assertNotEqual(uniform_metrics["brier"], uniform_metrics["brier_top1"])
+
+ # Synthetic records: 10 choice items, 10 score items
+ # Choice: 3 options, logits favor option 0 strongly, gold is option 0
+ choice_recs = [(0, [5.0, -2.0, -2.0], [1.0, 0.0, 0.0], 3) for _ in range(10)]
+ # Score: 5 levels, logits favor level 2, gold is level 2
+ score_recs = [(1, [-2.0, -2.0, 5.0, -2.0, -2.0], [0.0, 0.0, 1.0, 0.0, 0.0], 5) for _ in range(10)]
+ metrics = evaluate_records(choice_recs + score_recs)
+ self.assertEqual(metrics["items"], 20)
+ self.assertEqual(metrics["accuracy"], 1.0)
+ self.assertGreater(metrics["mean_confidence"], 0.9)
+ self.assertIsNotNone(metrics["ece"])
+ self.assertIsNotNone(metrics["brier"])
+ self.assertIsNotNone(metrics["brier_top1"])
+ self.assertIn("choice", metrics["by_type"])
+ self.assertIn("score", metrics["by_type"])
+ self.assertEqual(metrics["by_type"]["choice"]["items"], 10)
+ self.assertEqual(metrics["by_type"]["score"]["items"], 10)
+ self.assertIn("brier_top1", metrics["by_type"]["choice"])
+
+ # Temperature scaling softens confidences
+ warm = evaluate_records(choice_recs, temperature=[5.0, 5.0, 5.0])
+ self.assertLess(warm["mean_confidence"], metrics["by_type"]["choice"]["mean_confidence"])
+
def test_laya_train_cli_runs_on_csv(self):
csv_path = self.root / "tickets.csv"
lines = ["text,department"]
@@ -634,10 +696,32 @@ class EndToEndTests(unittest.TestCase):
self.assertTrue((out_dir / "model.safetensors").exists())
self.assertTrue((out_dir / "rl_agent_config.json").exists())
self.assertTrue((out_dir / "questions.json").exists())
+ self.assertTrue((out_dir / "train_report.json").exists())
+ self.assertTrue((out_dir / "checkpoint_latest" / "train_report.json").exists())
+ self.assertTrue((out_dir / "checkpoint_latest" / "rl_agent_config.json").exists())
+
+ out_cfg = json.loads((out_dir / "rl_agent_config.json").read_text(encoding="utf-8"))
+ latest_cfg = json.loads((out_dir / "checkpoint_latest" / "rl_agent_config.json").read_text(encoding="utf-8"))
+ # Top-level min_confidence ghost field must not be written to checkpoint config
+ self.assertNotIn("min_confidence", out_cfg)
+ # checkpoint_latest must be synchronized with final calibrated out_cfg
+ self.assertEqual(latest_cfg["temperature"], out_cfg["temperature"])
+ self.assertEqual(latest_cfg["training"]["train_report"], out_cfg["training"]["train_report"])
+
saved_q = json.loads((out_dir / "questions.json").read_text(encoding="utf-8"))
self.assertIn("label", saved_q)
self.assertEqual(saved_q["label"]["criteria"], ["billing", "technical"])
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertIsNone(report["eval_source"])
+ self.assertIsNone(report["eval_mode"])
+ self.assertFalse(report["is_held_out"])
+ self.assertEqual(report["eval_items"], 0)
+ self.assertIn("No evaluation performed", report["note"])
+ self.assertIsNone(report["before"])
+ self.assertIsNone(report["after"])
+ self.assertIsNone(report["comparison"])
+
agent = load(str(out_dir), device="cpu")
ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify",
"criteria": ["billing", "technical"]}})
@@ -653,6 +737,228 @@ class EndToEndTests(unittest.TestCase):
])
self.assertEqual(dry_code, 0)
+ def test_laya_train_cli_with_eval_file(self):
+ train_path = self.root / "train_eval_test.csv"
+ eval_path = self.root / "eval_eval_test.csv"
+ train_lines = ["text,department"]
+ for s in BILLING_STATES:
+ train_lines.append(f"{s},billing")
+ for s in TECH_STATES:
+ train_lines.append(f"{s},technical")
+ train_path.write_text("\n".join(train_lines), encoding="utf-8")
+
+ # Independent, non-overlapping evaluation items
+ eval_lines = [
+ "text,department",
+ "please dispute my credit card charge,billing",
+ "why was my card debited twice,billing",
+ "the API server returns 502 bad gateway,technical",
+ "the database socket connection timed out,technical",
+ ]
+ eval_path.write_text("\n".join(eval_lines), encoding="utf-8")
+
+ out_dir = self.root / "out_with_eval"
+ code = train_cli_main([
+ "--data", str(train_path),
+ "--eval", str(eval_path),
+ "--base", str(self.root / "base"),
+ "--out", str(out_dir),
+ "--text-column", "text",
+ "--label-column", "department",
+ "--epochs", "1",
+ "--micro-batch", "4",
+ "--grad-accum", "1",
+ "--loss", "soft-ce",
+ "--target-error", "0.08",
+ "--min-abstain-n", "2",
+ "--device", "cpu",
+ ])
+ self.assertEqual(code, 0)
+ self.assertTrue((out_dir / "train_report.json").exists())
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_source"], str(eval_path))
+ self.assertEqual(report["eval_mode"], "held_out")
+ self.assertTrue(report["is_held_out"])
+ self.assertIn("independent held-out evaluation", report["note"])
+ self.assertEqual(report["eval_items"], 4)
+ self.assertIn("before", report)
+ self.assertIn("after", report)
+ self.assertIn("comparison", report)
+ self.assertIn("brier", report["after"])
+ self.assertIn("brier_top1", report["after"])
+ self.assertIn("delta_brier", report["comparison"])
+ self.assertIn("delta_brier_top1", report["comparison"])
+
+ def test_laya_train_cli_with_overlapping_eval_file(self):
+ train_path = self.root / "train_overlap.csv"
+ eval_path = self.root / "eval_overlap.csv"
+ train_lines = ["text,department"]
+ eval_lines = ["text,department"]
+ for s in BILLING_STATES:
+ train_lines.append(f"{s},billing")
+ eval_lines.append(f"{s},billing")
+ for s in TECH_STATES:
+ train_lines.append(f"{s},technical")
+ eval_lines.append(f"{s},technical")
+ train_path.write_text("\n".join(train_lines), encoding="utf-8")
+ eval_path.write_text("\n".join(eval_lines), encoding="utf-8")
+
+ out_dir = self.root / "out_overlap"
+ code = train_cli_main([
+ "--data", str(train_path),
+ "--eval", str(eval_path),
+ "--base", str(self.root / "base"),
+ "--out", str(out_dir),
+ "--text-column", "text",
+ "--label-column", "department",
+ "--epochs", "1",
+ "--micro-batch", "4",
+ "--grad-accum", "1",
+ "--loss", "soft-ce",
+ "--device", "cpu",
+ ])
+ self.assertEqual(code, 0)
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_source"], str(eval_path))
+ self.assertEqual(report["eval_mode"], "overlapping_eval")
+ self.assertFalse(report["is_held_out"])
+ self.assertIn("overlap training data", report["note"])
+ self.assertIn("brier", report["after"])
+ self.assertIn("brier_top1", report["after"])
+ self.assertIn("delta_brier", report["comparison"])
+ self.assertIn("delta_brier_top1", report["comparison"])
+
+ def test_laya_train_cli_with_empty_or_skipped_eval_file(self):
+ train_path = self.root / "train_norm.csv"
+ eval_path = self.root / "eval_empty.jsonl"
+ train_lines = ["text,department"]
+ for s in BILLING_STATES:
+ train_lines.append(f"{s},billing")
+ for s in TECH_STATES:
+ train_lines.append(f"{s},technical")
+ train_path.write_text("\n".join(train_lines), encoding="utf-8")
+ eval_path.write_text("", encoding="utf-8")
+
+ out_dir = self.root / "out_empty_eval"
+ code = train_cli_main([
+ "--data", str(train_path),
+ "--eval", str(eval_path),
+ "--base", str(self.root / "base"),
+ "--out", str(out_dir),
+ "--text-column", "text",
+ "--label-column", "department",
+ "--epochs", "1",
+ "--micro-batch", "4",
+ "--grad-accum", "1",
+ "--loss", "soft-ce",
+ "--device", "cpu",
+ ])
+ self.assertEqual(code, 0)
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_source"], str(eval_path))
+ self.assertIsNone(report["eval_mode"])
+ self.assertFalse(report["is_held_out"])
+ self.assertEqual(report["eval_items"], 0)
+ self.assertIsNone(report["before"])
+ self.assertIsNone(report["after"])
+ self.assertIn("0 usable items", report["note"])
+
+ def test_overlap_detection_across_gold_and_expected_formats(self):
+ # Training row uses 'gold', eval row uses 'expected'
+ train_file = self.root / "train_gold.jsonl"
+ eval_file = self.root / "eval_expected.jsonl"
+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
+ train_row = {"state": "please refund payment", "questions": {"label": q_def}, "gold": {"label": {"probabilities": {"billing": 1.0}}}}
+ eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}
+ train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")
+ eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")
+
+ out_dir = self.root / "out_gold_expected_overlap"
+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))
+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_mode"], "overlapping_eval")
+ self.assertFalse(report["is_held_out"])
+ self.assertIn("overlap training data", report["note"])
+
+ def test_overlap_detection_with_different_targets(self):
+ # Training and eval have identical input text and question, but different target label
+ train_file = self.root / "train_diff_tgt.jsonl"
+ eval_file = self.root / "eval_diff_tgt.jsonl"
+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
+ train_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}
+ eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "technical"}}
+ train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")
+ eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")
+
+ out_dir = self.root / "out_diff_tgt_overlap"
+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))
+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_mode"], "overlapping_eval")
+ self.assertFalse(report["is_held_out"])
+ self.assertIn("overlap training data", report["note"])
+
+ def test_overlap_detection_with_calibration_slice(self):
+ # Training set has multiple items split into train and calib.
+ # Eval set only contains an item that ended up in the calibration slice, not train_items.
+ train_file = self.root / "train_calib_split.jsonl"
+ eval_file = self.root / "eval_calib_split.jsonl"
+ q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
+
+ states = ["refund please", "charged twice", "crash app", "screen error"]
+ rows = [
+ {"state": s, "questions": {"label": q_def}, "expected": {"label": "billing" if "refund" in s or "charged" in s else "technical"}}
+ for s in states
+ ]
+ train_file.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
+
+ import random
+ order = list(range(len(rows)))
+ random.Random(42).shuffle(order)
+ calib_row = rows[order[0]] # in calib set, not in train set
+ eval_file.write_text(json.dumps(calib_row) + "\n", encoding="utf-8")
+
+ out_dir = self.root / "out_calib_overlap"
+ cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, calib_frac=0.5, calib_max=2, calib_seed=42, eval_data=str(eval_file))
+ finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_mode"], "overlapping_eval")
+ self.assertFalse(report["is_held_out"])
+ self.assertIn("overlap calibration data", report["note"])
+
+ def test_laya_train_in_sample_calibration_mode(self):
+ csv_path = self.root / "calib_tickets.csv"
+ lines = ["text,department"]
+ for s in BILLING_STATES:
+ lines.append(f"{s},billing")
+ for s in TECH_STATES:
+ lines.append(f"{s},technical")
+ csv_path.write_text("\n".join(lines), encoding="utf-8")
+
+ out_dir = self.root / "out_in_sample"
+ cfg = TrainConfig(
+ text_column="text",
+ label_column="department",
+ epochs=1,
+ micro_batch=4,
+ grad_accum=1,
+ calib_frac=0.2,
+ calib_max=10,
+ target_error=0.10,
+ min_abstain_n=2,
+ )
+ finetune(str(csv_path), str(self.root / "base"), str(out_dir), cfg, device="cpu")
+ report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
+ self.assertEqual(report["eval_source"], "calibration_slice")
+ self.assertEqual(report["eval_mode"], "in_sample_calibration")
+ self.assertFalse(report["is_held_out"])
+ self.assertIn("in-sample calibration fit", report["note"])
+ self.assertGreater(report["eval_items"], 0)
+ self.assertIn("before", report)
+ self.assertIn("after", report)
+ self.assertIn("comparison", report)
+
class CalibrationReportTests(unittest.TestCase):
@staticmethod