Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
"""laya.train: data prep, option-order augmentation, losses and an end-to-end run.
CPU only and weight-free: the checkpoint is a one-layer BERT with a word-level tokenizer built
in a temp directory, so nothing is downloaded.
Run: python tests/test_train.py
"""
import json
import math
import os
from pathlib import Path
import random
import re
import sys
import tempfile
import unittest
import warnings
from unittest.mock import patch
os.environ.setdefault("USE_TF", "0")
os.environ.setdefault("USE_TORCH", "1")
os.environ.setdefault("HF_HUB_OFFLINE", "1")
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import numpy as np # noqa: E402
import torch # noqa: E402
from safetensors.torch import load_file, save_file # noqa: E402
from tokenizers import Tokenizer # noqa: E402
from tokenizers.models import WordLevel # noqa: E402
from tokenizers.pre_tokenizers import Whitespace # noqa: E402
from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
from laya import load # noqa: E402
from laya.common import ( # noqa: E402
DecisionModel,
QTYPES,
TEMP_MAX,
TEMP_MIN,
build_sequence,
proper_reward,
render_options,
unpermute_probs,
)
from laya.train import ( # noqa: E402
TrainConfig,
draw_option_order,
encode_item,
encode_state,
evaluate_records,
finetune,
items_from_rows,
make_item,
read_data,
rows_from_csv,
rlcd_loss,
soft_ce_loss,
split_calibration,
target_from_expected,
target_from_gold,
to_internal,
train_model,
)
from laya.train_cli import build_parser as build_train_parser, main as train_cli_main # noqa: E402
WORDS = ["refund", "invoice", "charged", "twice", "crash", "error", "app", "screen", "please",
"billing", "technical", "other", "is", "it", "urgent", "low", "high", "level", "question",
"choice", "score", "noul", "which", "team", "yes", "no", "the", "statement", "does",
"not", "hold", "holds", "one", "two", "three", "four", "five", "six", "seven", ":", ",", "?"]
DEPARTMENT = {"type": "choice", "instructions": "which team ?",
"criteria": {"billing": "", "technical": "", "other": ""}}
URGENT = {"type": "noul", "instructions": "is it urgent ?"}
LEVEL = {"type": "score", "instructions": "which level ?", "criteria": ["low", "high"]}
BILLING_STATES = ["please refund invoice", "charged twice invoice", "refund charged twice", "invoice refund please"]
TECH_STATES = ["app crash error", "screen error crash", "crash app screen", "error screen app"]
def make_tokenizer():
vocab = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}
for w in WORDS:
vocab[w] = len(vocab)
backend = Tokenizer(WordLevel(vocab, unk_token="[UNK]"))
backend.pre_tokenizer = Whitespace()
return PreTrainedTokenizerFast(tokenizer_object=backend, pad_token="[PAD]", unk_token="[UNK]",
cls_token="[CLS]", sep_token="[SEP]", mask_token="[MASK]")
def make_checkpoint(path: Path, tok):
torch.manual_seed(0)
config = BertConfig(vocab_size=len(tok), hidden_size=32, num_hidden_layers=1,
num_attention_heads=2, intermediate_size=64, max_position_embeddings=128)
config.save_pretrained(path / "encoder")
tok.save_pretrained(path / "tokenizer")
model = DecisionModel(BertModel(config), head_layers=1, n_act=2)
save_file(model.state_dict(), path / "model.safetensors")
cfg = {"encoder": "unused/offline", "head_layers": 1, "act_costs": {"act": 0},
"max_len": 64, "head_max_len": 40, "temperature": [1.0, 1.0, 1.0],
"temperature_by_options": {"choice:3-5": 0.7}}
(path / "rl_agent_config.json").write_text(json.dumps(cfg))
def rows(n_repeats=6):
out = []
for _ in range(n_repeats):
for s in BILLING_STATES:
out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},
"gold": {"department": {"probabilities": {"billing": 0.9, "technical": 0.05, "other": 0.05}},
"urgent": {"probabilities": {"false": 0.8, "true": 0.2}},
"level": {"probabilities": {"0": 0.7, "1": 0.3}}}})
for s in TECH_STATES:
out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},
"gold": {"department": {"probabilities": {"billing": 0.05, "technical": 0.9, "other": 0.05}},
"urgent": {"probabilities": {"false": 0.2, "true": 0.8}},
"level": {"probabilities": {"0": 0.2, "1": 0.8}}}})
return out
class DataTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tok = make_tokenizer()
def item(self, question, target, state="app crash", head_max_len=40):
item, reason = make_item(self.tok, to_internal("q", question), target,
encode_state(self.tok, state, 64), head_max_len)
self.assertIsNone(reason)
return item
def test_target_from_gold_orders_and_normalises(self):
def target(question, probs):
return target_from_gold(to_internal("q", question), {"probabilities": probs})
self.assertEqual(target(DEPARTMENT, {"technical": 2.0, "billing": 2.0}), [0.5, 0.5, 0.0])
self.assertEqual(target(URGENT, {"true": 0.75, "false": 0.25}), [0.25, 0.75])
self.assertEqual(target(LEVEL, {"1": 1.0}), [0.0, 1.0])
self.assertEqual(target(DEPARTMENT, {}), [1 / 3] * 3)
# list-form choice and non-string labels, which inference accepts
self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},
{"screen": 1.0}), [0.0, 1.0])
self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": {1: "", 2: ""}},
{"2": 3.0, "1": 1.0}), [0.25, 0.75])
for bad in ({"billing": -1.0}, {"billing": float("nan")}, {"billing": "high"}):
with self.assertRaises(ValueError):
target(DEPARTMENT, bad)
def test_target_from_expected_choice(self):
def target(question, exp, smoothing=0.0):
return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
self.assertEqual(target(DEPARTMENT, "billing"), [1.0, 0.0, 0.0])
self.assertEqual(target(DEPARTMENT, "technical"), [0.0, 1.0, 0.0])
self.assertEqual(target(DEPARTMENT, "other"), [0.0, 0.0, 1.0])
# Non-string labels matching keys
self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": {1: "a", 2: "b"}}, 1),
[1.0, 0.0])
def test_target_from_expected_noul(self):
def target(question, exp, smoothing=0.0):
return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
self.assertEqual(target(URGENT, True), [0.0, 1.0])
self.assertEqual(target(URGENT, False), [1.0, 0.0])
self.assertEqual(target(URGENT, "true"), [0.0, 1.0])
self.assertEqual(target(URGENT, "false"), [1.0, 0.0])
def test_target_from_expected_score(self):
def target(question, exp, smoothing=0.0):
return target_from_expected(to_internal("q", question), exp, label_smoothing=smoothing)
self.assertEqual(target(LEVEL, 0), [1.0, 0.0])
self.assertEqual(target(LEVEL, 1), [0.0, 1.0])
self.assertEqual(target(LEVEL, "1"), [0.0, 1.0])
# Fractional score answers (e.g. from laya-evals ScoreMAE / ScoreWithin)
three_level = {"type": "score", "instructions": "which level ?", "criteria": ["low", "mid", "high"]}
self.assertEqual(target(three_level, 1.5), [0.0, 0.5, 0.5])
res_frac2 = target(three_level, 0.2)
self.assertAlmostEqual(res_frac2[0], 0.8)
self.assertAlmostEqual(res_frac2[1], 0.2)
self.assertAlmostEqual(res_frac2[2], 0.0)
def test_target_from_expected_label_smoothing(self):
res = target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=0.1)
self.assertAlmostEqual(res[0], 0.9 + 0.1 / 3)
self.assertAlmostEqual(res[1], 0.1 / 3)
self.assertAlmostEqual(res[2], 0.1 / 3)
self.assertAlmostEqual(sum(res), 1.0)
res_noul = target_from_expected(to_internal("q", URGENT), True, label_smoothing=0.2)
self.assertAlmostEqual(res_noul[0], 0.1)
self.assertAlmostEqual(res_noul[1], 0.9)
self.assertAlmostEqual(sum(res_noul), 1.0)
def test_target_from_expected_rejects_invalid(self):
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", DEPARTMENT), "unknown_choice")
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", URGENT), "not_a_bool")
for bad_num in (-3, 7, 0.5, -0.1, 2):
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", URGENT), bad_num)
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", LEVEL), 99)
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", LEVEL), -0.5)
for bad_smooth in (-0.1, 1.0, 1.5, True):
with self.assertRaises(ValueError):
target_from_expected(to_internal("q", DEPARTMENT), "billing", label_smoothing=bad_smooth)
def test_resolve_checkpoint_dir(self):
from laya.train import resolve_checkpoint_dir
with tempfile.TemporaryDirectory() as tmpdir:
p = Path(tmpdir)
(p / "rl_agent_config.json").write_text("{}", encoding="utf-8")
self.assertEqual(resolve_checkpoint_dir(str(p)), str(p))
with self.assertRaises(FileNotFoundError):
resolve_checkpoint_dir(str(p / "nonexistent"))
def test_rows_from_csv_and_read_data(self):
with tempfile.TemporaryDirectory() as tmpdir:
csv_path = Path(tmpdir) / "tickets.csv"
csv_path.write_text("body,department\nrefund my invoice,billing\napp crash on startup,technical\n",
encoding="utf-8")
data = read_data(str(csv_path), text_column="body", label_column="department")
self.assertEqual(len(data), 2)
self.assertEqual(data[0]["state"], "refund my invoice")
self.assertEqual(data[0]["expected"]["label"], "billing")
q = data[0]["questions"]["label"]
self.assertEqual(q["type"], "choice")
self.assertEqual(q["criteria"], ["billing", "technical"])
# Test Excel UTF-8 BOM encoding
bom_csv = Path(tmpdir) / "bom.csv"
bom_csv.write_bytes("\ufeffbody,department\nrefund invoice,billing\ncrash,technical\n".encode("utf-8"))
bom_data = read_data(str(bom_csv), text_column="body", label_column="department")
self.assertEqual(len(bom_data), 2)
self.assertEqual(bom_data[0]["state"], "refund invoice")
def test_rows_from_csv_validation(self):
with tempfile.TemporaryDirectory() as tmpdir:
p = Path(tmpdir) / "bad.csv"
p.write_text("text,dept\nhi,billing\n", encoding="utf-8")
with self.assertRaises(ValueError):
rows_from_csv(str(p), text_column="body", label_column="dept")
p.write_text("text,dept\nhi,billing\nhello,billing\n", encoding="utf-8")
with self.assertRaises(ValueError):
rows_from_csv(str(p), text_column="text", label_column="dept")
def test_items_from_rows_supports_expected(self):
exp_rows = [
{"state": "please refund invoice", "questions": {"department": DEPARTMENT},
"expected": {"department": "billing"}},
{"state": "app crash", "questions": {"department": DEPARTMENT},
"expected": {"department": "technical"}},
]
items, skipped = items_from_rows(self.tok, exp_rows, 64, 40)
self.assertEqual(len(items), 2)
self.assertEqual(skipped, {})
self.assertEqual(items[0]["target"], [1.0, 0.0, 0.0])
self.assertEqual(items[1]["target"], [0.0, 1.0, 0.0])
def test_items_from_rows_skip_tracking(self):
exp_rows_with_empty = [
{"state": "", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},
{"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": ""}},
{"state": "valid text", "questions": {"department": DEPARTMENT}, "expected": {"department": "billing"}},
]
items, skipped = items_from_rows(self.tok, exp_rows_with_empty, 64, 40)
self.assertEqual(len(items), 1)
self.assertEqual(skipped.get("empty_text"), 1)
self.assertEqual(skipped.get("empty_label"), 1)
def test_questions_are_rendered_exactly_as_inference_renders_them(self):
# The sequence a training item produces must be the one `Agent` builds for the same
# question, or the model is trained on option texts it never sees.
from laya.agent import Agent
questions = [
DEPARTMENT, URGENT, LEVEL,
{"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},
{"type": "noul", "instructions": "is it ?", "labels": {"false": "no", "true": "yes"},
"criteria": {True: "holds", False: "does not hold"}},
{"type": "noul", "instructions": {"which": "team"}},
]
for question in questions:
with self.subTest(question=question):
k = len(render_options(Agent._to_internal(question)))
item = self.item(question, [1.0 / k] * k)
expected = build_sequence(self.tok, "app crash", Agent._to_internal(question), 64, 40)
enc = encode_item(self.tok, item, 64, 40)
self.assertEqual((enc["ids"], enc["markers"]), expected)
def test_rows_share_one_tokenized_state(self):
items, skipped = items_from_rows(self.tok, rows(1)[:1], 64, 40)
self.assertEqual((len(items), skipped), (3, {}))
self.assertTrue(items[0]["state_ids"] is items[1]["state_ids"] is items[2]["state_ids"])
self.assertEqual([it["qtype"] for it in items], [QTYPES["choice"], QTYPES["noul"], QTYPES["score"]])
def test_question_without_gold_is_not_an_item_or_a_skip(self):
row = dict(rows(1)[0])
row["gold"] = {"department": row["gold"]["department"]}
items, skipped = items_from_rows(self.tok, [row], 64, 40)
self.assertEqual((len(items), skipped), (1, {}))
def test_bad_rows_are_skipped_with_a_reason(self):
row = dict(rows(1)[0])
row["questions"] = dict(row["questions"],
broken={"type": "choice", "instructions": "which ?", "criteria": []},
negative=DEPARTMENT)
row["gold"] = dict(row["gold"], broken={"probabilities": {}},
negative={"probabilities": {"billing": -2.0}})
items, skipped = items_from_rows(self.tok, [row], 64, 40)
self.assertEqual((len(items), skipped), (3, {"invalid_question": 1, "invalid_target": 1}))
def test_options_the_head_budget_collapses_are_skipped(self):
# Two options that only differ after the per-option cap come out as the same span (#538).
q = to_internal("q", {"type": "choice", "instructions": "which ?",
"criteria": {"one two three four five six": "", "one two three four five seven": ""}})
state_ids = encode_state(self.tok, "app", 64)
self.assertEqual(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=20),
(None, "options_collapsed"))
self.assertIsNotNone(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=64)[0])
def test_options_past_max_len_are_skipped_like_inference_refuses_them(self):
# With a head longer than max_len, build_sequence drops the last options' markers and the
# batch would crash on a target longer than its markers. Agent refuses the same question.
from laya.common import build_head
question = {"type": "choice", "instructions": "which ?",
"criteria": ["one", "two", "three", "four", "five", "six", "seven"]}
q = to_internal("q", question)
_ids, markers, _stats = build_head(self.tok, q, 64)
state_ids = encode_state(self.tok, "app", 64)
target = [1.0 / 7] * 7
self.assertEqual(make_item(self.tok, q, target, state_ids, 64, max_len=markers[-1]),
(None, "options_beyond_max_len"))
self.assertIsNotNone(make_item(self.tok, q, target, state_ids, 64, max_len=markers[-1] + 1)[0])
self.assertIsNotNone(make_item(self.tok, q, target, state_ids, 64)[0]) # no max_len: not checked
row = {"state": "app", "questions": {"q": question},
"gold": {"q": {"probabilities": {"one": 1.0}}}}
self.assertEqual(items_from_rows(self.tok, [row], markers[-1], 64), ([], {"options_beyond_max_len": 1}))
# the same budget really does cut a marker in the sequence training would build
_seq, kept = build_sequence(self.tok, "app", q, markers[-1], 64)
self.assertEqual(len(kept), 6)
def test_target_length_must_match_options(self):
state_ids = encode_state(self.tok, "app", 64)
self.assertEqual(make_item(self.tok, to_internal("q", DEPARTMENT), [0.5, 0.5], state_ids, 40),
(None, "target_mismatch"))
def test_canonical_encoding_matches_build_sequence(self):
item = self.item(DEPARTMENT, [0.8, 0.1, 0.1])
enc = encode_item(self.tok, item, 64, 40)
ids, markers = build_sequence(self.tok, "app crash", item["q"], 64, 40)
self.assertEqual((enc["ids"], enc["markers"], enc["target"]), (ids, markers, [0.8, 0.1, 0.1]))
def test_permuted_item_carries_a_permuted_target(self):
item = self.item(DEPARTMENT, [0.7, 0.2, 0.1])
order = [2, 0, 1]
enc = encode_item(self.tok, item, 64, 40, option_order=order)
self.assertEqual(enc["target"], [0.1, 0.7, 0.2])
labels = list(DEPARTMENT["criteria"])
# The token right after slot s's [MASK] is the label of option order[s].
for slot, marker in enumerate(enc["markers"]):
self.assertEqual(enc["ids"][marker], self.tok.mask_token_id)
self.assertEqual(enc["ids"][marker + 1], self.tok.convert_tokens_to_ids(labels[order[slot]]))
def test_permuted_training_input_is_what_inference_scores_for_that_order(self):
# Inference with option_order builds the same tokens in the same slots, and
# unpermute_probs maps the slot-ordered target back to the canonical one.
from laya.agent import Agent
order = [1, 2, 0]
item = self.item(DEPARTMENT, [0.6, 0.3, 0.1])
enc = encode_item(self.tok, item, 64, 40, option_order=order)
internal = Agent._to_internal(dict(DEPARTMENT, option_order=order))
expected_ids, _ = build_sequence(self.tok, "app crash", internal, 64, 40,
option_order=internal["option_order"])
self.assertEqual(enc["ids"], expected_ids)
self.assertEqual(list(unpermute_probs(np.array(enc["target"]), order)), item["target"])
def test_only_configured_types_are_shuffled(self):
state_ids = encode_state(self.tok, "app", 64)
choice, _ = make_item(self.tok, to_internal("q", DEPARTMENT), [1.0, 0.0, 0.0], state_ids, 40)
noul, _ = make_item(self.tok, to_internal("q", URGENT), [0.5, 0.5], state_ids, 40)
self.assertIsNone(draw_option_order(noul, random.Random(0), ("choice",)))
orders = {tuple(draw_option_order(choice, random.Random(s), ("choice",))) for s in range(20)}
self.assertTrue(all(sorted(o) == [0, 1, 2] for o in orders))
self.assertGreater(len(orders), 1)
self.assertEqual(draw_option_order(choice, random.Random(3), ("choice",)),
draw_option_order(choice, random.Random(3), ("choice",)))
self.assertIsNone(draw_option_order(choice, random.Random(0), ()))
def test_calibration_split_is_disjoint_and_seeded(self):
items = list(range(50))
train, calib = split_calibration(items, calib_max=400, calib_frac=0.1, seed=7)
self.assertEqual((len(train), len(calib)), (45, 5))
self.assertEqual(sorted(train + calib), items)
self.assertEqual(split_calibration(items, 400, 0.1, 7), (train, calib))
self.assertEqual(len(split_calibration(items, 3, 0.5, 7)[1]), 3)
class LossTests(unittest.TestCase):
def test_soft_ce_ignores_padded_options(self):
logits = torch.tensor([[2.0, 0.5, 9.0], [1.0, -1.0, 0.0]])
target = torch.tensor([[0.75, 0.25, 0.0], [0.2, 0.3, 0.5]])
mask = torch.tensor([[True, True, False], [True, True, True]])
manual = -(torch.tensor([0.75, 0.25]) * torch.log_softmax(torch.tensor([2.0, 0.5]), -1)).sum()
manual = (manual - (target[1] * torch.log_softmax(logits[1], -1)).sum()) / 2
self.assertAlmostEqual(soft_ce_loss(logits, target, mask).item(), manual.item(), places=5)
def test_rlcd_is_finite_and_differentiable(self):
torch.manual_seed(0)
logits = torch.randn(4, 3, requires_grad=True)
target = torch.softmax(torch.randn(4, 3), -1)
mask = torch.ones(4, 3, dtype=torch.bool)
loss = rlcd_loss(logits, target, mask, torch.tensor([0, 1, 2, 0]), sigma=0.4)
loss.backward()
self.assertTrue(torch.isfinite(loss))
self.assertTrue(torch.isfinite(logits.grad).all())
def test_rlcd_matches_the_notebook_objective(self):
# The loss as written in research/scripts/finetune_single_device.py, inlined: with the
# same RNG state, rlcd_loss must give the same value and the same gradient.
def script_loss(logits, target, mask, qtype, sigma):
k = mask.sum(-1, keepdim=True).float()
eps = torch.randn((4,) + logits.shape) * sigma * mask
eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
z = logits.detach().unsqueeze(0) + eps
q = torch.softmax(z.masked_fill(~mask, -1e4), -1)
with torch.no_grad():
r = proper_reward(q, target.unsqueeze(0), qtype, mask, w_sph=0.75, w_rps=1.0)
adv = r - r.mean(0, keepdim=True)
adv = adv / (adv.std() + 1e-6)
logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)
ce = (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()
return -(adv * logp).mean() - ce
torch.manual_seed(1)
base = torch.randn(5, 4)
mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 0, 0]],
dtype=torch.bool)
target = torch.softmax(torch.randn(5, 4), -1) * mask
target = target / target.sum(-1, keepdim=True)
qtype = torch.tensor([0, 2, 1, 0, 2])
results = []
for fn in (lambda lg: script_loss(lg, target, mask, qtype, 0.3),
lambda lg: rlcd_loss(lg, target, mask, qtype, 0.3)):
logits = base.masked_fill(~mask, -1e4).clone().requires_grad_(True)
torch.manual_seed(5)
loss = fn(logits)
loss.backward()
results.append((loss.item(), logits.grad.clone()))
self.assertAlmostEqual(results[0][0], results[1][0], places=5)
self.assertTrue(torch.allclose(results[0][1], results[1][1], atol=1e-6))
def test_defaults_train_like_the_notebook(self):
config = TrainConfig()
self.assertEqual((config.loss, config.shuffle_options), ("rlcd", ()))
self.assertEqual(config.micro_batch * config.grad_accum, 64)
def test_config_rejects_unknown_values(self):
for bad in (TrainConfig(loss="mse"), TrainConfig(shuffle_options=("choise",)),
TrainConfig(epochs=0), TrainConfig(calib_frac=1.0)):
with self.assertRaises(ValueError):
bad.validate()
class TrainingLoopTests(unittest.TestCase):
def test_partial_accumulation_window_averages_its_own_steps(self):
class ScalarModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.encoder = torch.nn.Identity()
self.value = torch.nn.Parameter(torch.tensor(0.0))
model = ScalarModel()
tok = type("Tokenizer", (), {"pad_token_id": 0})()
item = {"q": {"t": "choice"}, "k": 2}
batch = {
"marker_mask": torch.ones((1, 1), dtype=torch.bool),
"target": torch.ones((1, 1)),
}
config = TrainConfig(
epochs=1,
micro_batch=1,
grad_accum=2,
head_lr=1.0,
min_lr=1.0,
weight_decay=0.0,
grad_clip=100.0,
loss="soft-ce",
freeze_encoder=True,
amp=False,
log_every=0,
)
with patch("laya.train.encode_item", return_value={}), \
patch("laya.train.collate_items", return_value=batch), \
patch("laya.train._forward", side_effect=lambda current, *_args: current.value), \
patch("laya.train.soft_ce_loss", side_effect=lambda logits, *_args: logits), \
patch("laya.train.torch.optim.AdamW",
side_effect=lambda groups, weight_decay: torch.optim.SGD(
groups, weight_decay=weight_decay)):
train_model(model, tok, [item] * 3, config, torch.device("cpu"), 1, 1)
# The full two-step window and the final one-step window each contribute one update.
self.assertAlmostEqual(model.value.item(), -2.0)
class EndToEndTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmp = tempfile.TemporaryDirectory()
cls.addClassCleanup(cls.tmp.cleanup)
cls.root = Path(cls.tmp.name)
cls.tok = make_tokenizer()
make_checkpoint(cls.root / "base", cls.tok)
cls.data = cls.root / "train.jsonl"
cls.data.write_text("\n".join(json.dumps(r) for r in rows()), encoding="utf-8")
def run_finetune(self, out, **overrides):
# Options stay in order here: a randomly initialised one-layer encoder cannot learn to read
# option labels in a handful of epochs, only their slots. The permutation itself is
# covered by DataTests; test_shuffled_run_saves_a_loadable_checkpoint runs it end to end.
kwargs = dict(epochs=12, micro_batch=8, grad_accum=1, encoder_lr=1e-3, head_lr=1e-3,
calib_frac=0.2, log_every=0, loss="soft-ce", shuffle_options=())
kwargs.update(overrides)
config = TrainConfig(**kwargs)
with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
return finetune(str(self.data), str(self.root / "base"), str(self.root / out), config, device="cpu")
def test_finetune_learns_and_saves_a_loadable_checkpoint(self):
report = self.run_finetune("out")
self.assertEqual(report["train_items"] + report["calibration_items"], len(rows()) * 3)
self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])
saved = json.loads((self.root / "out" / "rl_agent_config.json").read_text())
self.assertTrue(saved["fine_tuned"])
self.assertEqual((saved["max_len"], saved["head_max_len"]), (64, 40))
# The inherited bucket would override the new fit at inference; too few items to refit it.
self.assertNotIn("temperature_by_options", saved)
self.assertTrue(all(TEMP_MIN <= t <= TEMP_MAX for t in saved["temperature"]))
self.assertEqual(saved["training"]["laya_train"]["loss"], "soft-ce")
self.assertTrue((self.root / "out" / "checkpoint_latest" / "model.safetensors").exists())
with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
agent = load(str(self.root / "out"), device="cpu")
answers = agent.predict("please refund invoice", {"department": DEPARTMENT})["answers"]
self.assertEqual(answers["department"]["choice"], "billing")
answers = agent.predict("app crash error", {"department": DEPARTMENT})["answers"]
self.assertEqual(answers["department"]["choice"], "technical")
def test_shuffled_run_saves_a_loadable_checkpoint(self):
report = self.run_finetune("out_shuffled", shuffle_options=("choice",), epochs=1)
saved = json.loads((self.root / "out_shuffled" / "rl_agent_config.json").read_text())
self.assertEqual(saved["training"]["laya_train"]["shuffle_options"], ["choice"])
self.assertEqual(len(report["epoch_loss"]), 1)
agent = load(str(self.root / "out_shuffled"), device="cpu")
self.assertIn(agent.predict("app", {"department": DEPARTMENT})["answers"]["department"]["choice"],
DEPARTMENT["criteria"])
def test_startup_prints_the_optimizer_budget(self):
import contextlib
import io
from laya.train import optimizer_updates
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
report = self.run_finetune("out_budget", epochs=2)
expected = optimizer_updates(report["train_items"], 8, 1, 2)
self.assertIn("optimizer budget", buf.getvalue())
self.assertIn("%d optimizer updates" % expected, buf.getvalue())
def test_rlcd_objective_still_trains(self):
report = self.run_finetune("out_rlcd", loss="rlcd")
self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])
def test_frozen_encoder_is_left_untouched(self):
self.run_finetune("out_frozen", freeze_encoder=True, epochs=2)
before = load_file(str(self.root / "base" / "model.safetensors"))
after = load_file(str(self.root / "out_frozen" / "model.safetensors"))
for name in before:
if name.startswith("encoder."):
self.assertTrue(torch.allclose(before[name].half().float(), after[name].float()), name)
self.assertFalse(torch.allclose(before["scorer.1.weight"].half().float(), after["scorer.1.weight"].float()))
def test_model_is_on_the_device_before_the_base_evaluation(self):
# finetune scores the base checkpoint before train_model moves the model, which crashed
# on CUDA and MPS (inputs on the device, weights on CPU). CPU cannot show the crash, so
# record the order instead: the model must be moved before the first calibration_records.
import laya.train as lt
events = []
real_load, real_records = lt.load_checkpoint, lt.calibration_records
def load_checkpoint(model_dir):
model, tok, cfg = real_load(model_dir)
real_to = model.to
def to(*args, **kwargs):
events.append(("to", torch.device(args[0] if args else kwargs["device"])))
return real_to(*args, **kwargs)
model.to = to
return model, tok, cfg
def calibration_records(model, tok, items, device, *args, **kwargs):
events.append(("records", device))
return real_records(model, tok, items, device, *args, **kwargs)
with patch("laya.train.load_checkpoint", load_checkpoint), \
patch("laya.train.calibration_records", calibration_records):
self.run_finetune("out_device", epochs=1)
cpu = torch.device("cpu")
first_eval = events.index(("records", cpu))
self.assertIn(("to", cpu), events[:first_eval], events)
def test_frozen_encoder_runs_without_dropout(self):
from laya.train import load_checkpoint
model, tok, _cfg = load_checkpoint(str(self.root / "base"))
modes = []
model.encoder.register_forward_pre_hook(lambda module, _args: modes.append(module.training))
items, _ = items_from_rows(tok, rows(1)[:4], 64, 40)
train_model(model, tok, items, TrainConfig(epochs=1, freeze_encoder=True, log_every=0),
torch.device("cpu"), 64, 40)
self.assertTrue(modes)
self.assertFalse(any(modes))
def test_train_model_rejects_empty_input(self):
with self.assertRaises(ValueError):
train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)
def test_laya_train_cli_parser_and_validation(self):
parser = build_train_parser()
args = parser.parse_args(["--data", "train.csv", "--eval", "eval.csv", "--target-error", "0.05",
"--min-abstain-n", "5", "--out", "./out", "--label-smoothing", "0.1",
"--shuffle-options", "--freeze-encoder"])
self.assertEqual(args.data, "train.csv")
self.assertEqual(args.eval_data, "eval.csv")
self.assertEqual(args.target_error, 0.05)
self.assertEqual(args.min_abstain_n, 5)
self.assertEqual(args.output_dir, "./out")
self.assertEqual(args.label_smoothing, 0.1)
self.assertTrue(args.shuffle_options)
self.assertTrue(args.freeze_encoder)
# --out is required when --dry-run is not set
code_missing_out = train_cli_main(["--data", "train.csv"])
self.assertEqual(code_missing_out, 2)
# --dry-run without --out succeeds parsing
dry_args = parser.parse_args(["--data", "train.csv", "--dry-run"])
self.assertIsNone(dry_args.output_dir)
self.assertTrue(dry_args.dry_run)
def test_train_config_eval_validation(self):
cfg = TrainConfig(eval_data="val.csv", target_error=0.15, min_abstain_n=20)
cfg.validate()
with self.assertRaises(ValueError):
TrainConfig(target_error=-0.1).validate()
with self.assertRaises(ValueError):
TrainConfig(target_error=1.5).validate()
with self.assertRaises(ValueError):
TrainConfig(target_error=True).validate()
with self.assertRaises(ValueError):
TrainConfig(min_abstain_n=0).validate()
with self.assertRaises(ValueError):
TrainConfig(min_abstain_n=-5).validate()
with self.assertRaises(ValueError):
TrainConfig(eval_data=123).validate()
def test_evaluate_records(self):
# Empty records returns default dict
empty = evaluate_records([])
self.assertEqual(empty["items"], 0)
self.assertIsNone(empty["ece"])
self.assertIsNone(empty["brier"])
self.assertIsNone(empty["brier_top1"])
# Distinct Brier values for known input:
# 3 options, uniform prediction (logits [0, 0, 0] -> p = [1/3, 1/3, 1/3]), one-hot target [1, 0, 0].
# Multiclass Brier = (1/3 - 1)^2 + (1/3 - 0)^2 + (1/3 - 0)^2 = 4/9 + 1/9 + 1/9 = 6/9 = 0.6667
# Top-1 Confidence Brier = (1/3 - 1)^2 = 4/9 = 0.4444
uniform_recs = [(0, [0.0, 0.0, 0.0], [1.0, 0.0, 0.0], 3)]
uniform_metrics = evaluate_records(uniform_recs)
self.assertEqual(uniform_metrics["brier"], 0.6667)
self.assertEqual(uniform_metrics["brier_top1"], 0.4444)
self.assertNotEqual(uniform_metrics["brier"], uniform_metrics["brier_top1"])
# Synthetic records: 10 choice items, 10 score items
# Choice: 3 options, logits favor option 0 strongly, gold is option 0
choice_recs = [(0, [5.0, -2.0, -2.0], [1.0, 0.0, 0.0], 3) for _ in range(10)]
# Score: 5 levels, logits favor level 2, gold is level 2
score_recs = [(1, [-2.0, -2.0, 5.0, -2.0, -2.0], [0.0, 0.0, 1.0, 0.0, 0.0], 5) for _ in range(10)]
metrics = evaluate_records(choice_recs + score_recs)
self.assertEqual(metrics["items"], 20)
self.assertEqual(metrics["accuracy"], 1.0)
self.assertGreater(metrics["mean_confidence"], 0.9)
self.assertIsNotNone(metrics["ece"])
self.assertIsNotNone(metrics["brier"])
self.assertIsNotNone(metrics["brier_top1"])
self.assertIn("choice", metrics["by_type"])
self.assertIn("score", metrics["by_type"])
self.assertEqual(metrics["by_type"]["choice"]["items"], 10)
self.assertEqual(metrics["by_type"]["score"]["items"], 10)
self.assertIn("brier_top1", metrics["by_type"]["choice"])
# Temperature scaling softens confidences
warm = evaluate_records(choice_recs, temperature=[5.0, 5.0, 5.0])
self.assertLess(warm["mean_confidence"], metrics["by_type"]["choice"]["mean_confidence"])
def test_laya_train_cli_runs_on_csv(self):
csv_path = self.root / "tickets.csv"
lines = ["text,department"]
for s in BILLING_STATES:
lines.append(f"{s},billing")
for s in TECH_STATES:
lines.append(f"{s},technical")
csv_path.write_text("\n".join(lines), encoding="utf-8")
out_dir = self.root / "out_cli"
code = train_cli_main([
"--data", str(csv_path),
"--base", str(self.root / "base"),
"--out", str(out_dir),
"--text-column", "text",
"--label-column", "department",
"--epochs", "1",
"--micro-batch", "4",
"--grad-accum", "1",
"--loss", "soft-ce",
"--label-smoothing", "0.05",
"--device", "cpu",
])
self.assertEqual(code, 0)
self.assertTrue((out_dir / "model.safetensors").exists())
self.assertTrue((out_dir / "rl_agent_config.json").exists())
self.assertTrue((out_dir / "questions.json").exists())
self.assertTrue((out_dir / "train_report.json").exists())
self.assertTrue((out_dir / "checkpoint_latest" / "train_report.json").exists())
self.assertTrue((out_dir / "checkpoint_latest" / "rl_agent_config.json").exists())
out_cfg = json.loads((out_dir / "rl_agent_config.json").read_text(encoding="utf-8"))
latest_cfg = json.loads((out_dir / "checkpoint_latest" / "rl_agent_config.json").read_text(encoding="utf-8"))
# Top-level min_confidence ghost field must not be written to checkpoint config
self.assertNotIn("min_confidence", out_cfg)
# checkpoint_latest must be synchronized with final calibrated out_cfg
self.assertEqual(latest_cfg["temperature"], out_cfg["temperature"])
self.assertEqual(latest_cfg["training"]["train_report"], out_cfg["training"]["train_report"])
saved_q = json.loads((out_dir / "questions.json").read_text(encoding="utf-8"))
self.assertIn("label", saved_q)
self.assertEqual(saved_q["label"]["criteria"], ["billing", "technical"])
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertIsNone(report["eval_source"])
self.assertIsNone(report["eval_mode"])
self.assertFalse(report["is_held_out"])
self.assertEqual(report["eval_items"], 0)
self.assertIn("No evaluation performed", report["note"])
self.assertIsNone(report["before"])
self.assertIsNone(report["after"])
self.assertIsNone(report["comparison"])
agent = load(str(out_dir), device="cpu")
ans = agent.predict("please refund invoice", {"label": {"type": "choice", "instructions": "Classify",
"criteria": ["billing", "technical"]}})
self.assertIn(ans["answers"]["label"]["choice"], ["billing", "technical"])
# Test --dry-run
dry_code = train_cli_main([
"--data", str(csv_path),
"--base", str(self.root / "base"),
"--text-column", "text",
"--label-column", "department",
"--dry-run",
])
self.assertEqual(dry_code, 0)
def test_laya_train_cli_with_eval_file(self):
train_path = self.root / "train_eval_test.csv"
eval_path = self.root / "eval_eval_test.csv"
train_lines = ["text,department"]
for s in BILLING_STATES:
train_lines.append(f"{s},billing")
for s in TECH_STATES:
train_lines.append(f"{s},technical")
train_path.write_text("\n".join(train_lines), encoding="utf-8")
# Independent, non-overlapping evaluation items
eval_lines = [
"text,department",
"please dispute my credit card charge,billing",
"why was my card debited twice,billing",
"the API server returns 502 bad gateway,technical",
"the database socket connection timed out,technical",
]
eval_path.write_text("\n".join(eval_lines), encoding="utf-8")
out_dir = self.root / "out_with_eval"
code = train_cli_main([
"--data", str(train_path),
"--eval", str(eval_path),
"--base", str(self.root / "base"),
"--out", str(out_dir),
"--text-column", "text",
"--label-column", "department",
"--epochs", "1",
"--micro-batch", "4",
"--grad-accum", "1",
"--loss", "soft-ce",
"--target-error", "0.08",
"--min-abstain-n", "2",
"--device", "cpu",
])
self.assertEqual(code, 0)
self.assertTrue((out_dir / "train_report.json").exists())
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_source"], str(eval_path))
self.assertEqual(report["eval_mode"], "held_out")
self.assertTrue(report["is_held_out"])
self.assertIn("independent held-out evaluation", report["note"])
self.assertEqual(report["eval_items"], 4)
self.assertIn("before", report)
self.assertIn("after", report)
self.assertIn("comparison", report)
self.assertIn("brier", report["after"])
self.assertIn("brier_top1", report["after"])
self.assertIn("delta_brier", report["comparison"])
self.assertIn("delta_brier_top1", report["comparison"])
def test_laya_train_cli_with_overlapping_eval_file(self):
train_path = self.root / "train_overlap.csv"
eval_path = self.root / "eval_overlap.csv"
train_lines = ["text,department"]
eval_lines = ["text,department"]
for s in BILLING_STATES:
train_lines.append(f"{s},billing")
eval_lines.append(f"{s},billing")
for s in TECH_STATES:
train_lines.append(f"{s},technical")
eval_lines.append(f"{s},technical")
train_path.write_text("\n".join(train_lines), encoding="utf-8")
eval_path.write_text("\n".join(eval_lines), encoding="utf-8")
out_dir = self.root / "out_overlap"
code = train_cli_main([
"--data", str(train_path),
"--eval", str(eval_path),
"--base", str(self.root / "base"),
"--out", str(out_dir),
"--text-column", "text",
"--label-column", "department",
"--epochs", "1",
"--micro-batch", "4",
"--grad-accum", "1",
"--loss", "soft-ce",
"--device", "cpu",
])
self.assertEqual(code, 0)
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_source"], str(eval_path))
self.assertEqual(report["eval_mode"], "overlapping_eval")
self.assertFalse(report["is_held_out"])
self.assertIn("overlap training data", report["note"])
self.assertIn("brier", report["after"])
self.assertIn("brier_top1", report["after"])
self.assertIn("delta_brier", report["comparison"])
self.assertIn("delta_brier_top1", report["comparison"])
def test_laya_train_cli_with_empty_or_skipped_eval_file(self):
train_path = self.root / "train_norm.csv"
eval_path = self.root / "eval_empty.jsonl"
train_lines = ["text,department"]
for s in BILLING_STATES:
train_lines.append(f"{s},billing")
for s in TECH_STATES:
train_lines.append(f"{s},technical")
train_path.write_text("\n".join(train_lines), encoding="utf-8")
eval_path.write_text("", encoding="utf-8")
out_dir = self.root / "out_empty_eval"
code = train_cli_main([
"--data", str(train_path),
"--eval", str(eval_path),
"--base", str(self.root / "base"),
"--out", str(out_dir),
"--text-column", "text",
"--label-column", "department",
"--epochs", "1",
"--micro-batch", "4",
"--grad-accum", "1",
"--loss", "soft-ce",
"--device", "cpu",
])
self.assertEqual(code, 0)
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_source"], str(eval_path))
self.assertIsNone(report["eval_mode"])
self.assertFalse(report["is_held_out"])
self.assertEqual(report["eval_items"], 0)
self.assertIsNone(report["before"])
self.assertIsNone(report["after"])
self.assertIn("0 usable items", report["note"])
def test_overlap_detection_across_gold_and_expected_formats(self):
# Training row uses 'gold', eval row uses 'expected'
train_file = self.root / "train_gold.jsonl"
eval_file = self.root / "eval_expected.jsonl"
q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
train_row = {"state": "please refund payment", "questions": {"label": q_def}, "gold": {"label": {"probabilities": {"billing": 1.0}}}}
eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}
train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")
eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")
out_dir = self.root / "out_gold_expected_overlap"
cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))
finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_mode"], "overlapping_eval")
self.assertFalse(report["is_held_out"])
self.assertIn("overlap training data", report["note"])
def test_overlap_detection_with_different_targets(self):
# Training and eval have identical input text and question, but different target label
train_file = self.root / "train_diff_tgt.jsonl"
eval_file = self.root / "eval_diff_tgt.jsonl"
q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
train_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "billing"}}
eval_row = {"state": "please refund payment", "questions": {"label": q_def}, "expected": {"label": "technical"}}
train_file.write_text(json.dumps(train_row) + "\n", encoding="utf-8")
eval_file.write_text(json.dumps(eval_row) + "\n", encoding="utf-8")
out_dir = self.root / "out_diff_tgt_overlap"
cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, eval_data=str(eval_file))
finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_mode"], "overlapping_eval")
self.assertFalse(report["is_held_out"])
self.assertIn("overlap training data", report["note"])
def test_overlap_detection_with_calibration_slice(self):
# Training set has multiple items split into train and calib.
# Eval set only contains an item that ended up in the calibration slice, not train_items.
train_file = self.root / "train_calib_split.jsonl"
eval_file = self.root / "eval_calib_split.jsonl"
q_def = {"type": "choice", "instructions": "Classify", "criteria": ["billing", "technical"]}
states = ["refund please", "charged twice", "crash app", "screen error"]
rows = [
{"state": s, "questions": {"label": q_def}, "expected": {"label": "billing" if "refund" in s or "charged" in s else "technical"}}
for s in states
]
train_file.write_text("\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8")
import random
order = list(range(len(rows)))
random.Random(42).shuffle(order)
calib_row = rows[order[0]] # in calib set, not in train set
eval_file.write_text(json.dumps(calib_row) + "\n", encoding="utf-8")
out_dir = self.root / "out_calib_overlap"
cfg = TrainConfig(epochs=1, micro_batch=1, grad_accum=1, calib_frac=0.5, calib_max=2, calib_seed=42, eval_data=str(eval_file))
finetune(str(train_file), str(self.root / "base"), str(out_dir), cfg, device="cpu")
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_mode"], "overlapping_eval")
self.assertFalse(report["is_held_out"])
self.assertIn("overlap calibration data", report["note"])
def test_laya_train_in_sample_calibration_mode(self):
csv_path = self.root / "calib_tickets.csv"
lines = ["text,department"]
for s in BILLING_STATES:
lines.append(f"{s},billing")
for s in TECH_STATES:
lines.append(f"{s},technical")
csv_path.write_text("\n".join(lines), encoding="utf-8")
out_dir = self.root / "out_in_sample"
cfg = TrainConfig(
text_column="text",
label_column="department",
epochs=1,
micro_batch=4,
grad_accum=1,
calib_frac=0.2,
calib_max=10,
target_error=0.10,
min_abstain_n=2,
)
finetune(str(csv_path), str(self.root / "base"), str(out_dir), cfg, device="cpu")
report = json.loads((out_dir / "train_report.json").read_text(encoding="utf-8"))
self.assertEqual(report["eval_source"], "calibration_slice")
self.assertEqual(report["eval_mode"], "in_sample_calibration")
self.assertFalse(report["is_held_out"])
self.assertIn("in-sample calibration fit", report["note"])
self.assertGreater(report["eval_items"], 0)
self.assertIn("before", report)
self.assertIn("after", report)
self.assertIn("comparison", report)
class CalibrationReportTests(unittest.TestCase):
@staticmethod
def records(qtype, n, k=3, scale=1.0):
# Logits that favour the target option by `scale`; target one-hot on option 0.
return [(qtype, [scale] + [0.0] * (k - 1), [1.0] + [0.0] * (k - 1), k) for _ in range(n)]
def test_each_kind_of_weak_fit_is_named(self):
from laya.train import CALIB_WARN_N, calibration_report
records = (self.records(QTYPES["choice"], 5) + self.records(QTYPES["noul"], 30, k=2)
+ self.records(QTYPES["score"], CALIB_WARN_N + 10))
report = calibration_report(records, [1.0, 1.0, 2.0]) # choice, score, noul
self.assertEqual(report["choice"]["items"], 5)
self.assertIn("not fitted", report["choice"]["issues"][0])
self.assertEqual(len(report["noul"]["issues"]), 1)
self.assertIn("only 30", report["noul"]["issues"][0])
self.assertEqual(len(report["score"]["issues"]), 1)
self.assertIn("unchanged at 1.0", report["score"]["issues"][0])
self.assertAlmostEqual(report["score"]["accuracy"], 1.0)
def test_clamped_and_healthy_fits(self):
from laya.train import CALIB_WARN_N, calibration_report
n = CALIB_WARN_N
records = self.records(QTYPES["choice"], n) + self.records(QTYPES["score"], n)
report = calibration_report(records, [TEMP_MIN, 1.3, 1.0]) # choice, score, noul
self.assertIn("clamp", report["choice"]["issues"][0])
self.assertEqual(report["score"]["issues"], [])
# a type absent from the data is reported, but not as a problem
self.assertEqual((report["noul"]["items"], report["noul"]["issues"]), (0, []))
self.assertNotIn("accuracy", report["noul"])
p = torch.softmax(torch.tensor([1.0, 0.0, 0.0]) / 1.3, -1)
self.assertAlmostEqual(report["score"]["mean_confidence"], p.max().item(), places=6)
def test_finetune_warns_and_records_a_weak_calibration(self):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
make_checkpoint(root / "base", make_tokenizer())
config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)
# The rows go straight to finetune through read_jsonl rather than through a file: nothing
# here needs the JSONL parsing, which the end-to-end tests above already cover.
# warnings.catch_warnings, not assertWarns: assertWarns walks every loaded module, and
# transformers' lazy modules then try to import optional packages such as torchvision.
with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")), \
patch("laya.train.read_jsonl", return_value=rows(2)), \
warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
report = finetune("rows.jsonl", str(root / "base"), str(root / "out"), config, device="cpu")
saved = json.loads((root / "out" / "rl_agent_config.json").read_text())
messages = [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]
self.assertTrue(any(re.match(r"laya\.train: \w+ calibration: not fitted", m) for m in messages), messages)
self.assertEqual(saved["training"]["laya_train_calibration"], report["calibration"])
self.assertEqual(sum(e["items"] for e in report["calibration"].values()), report["calibration_items"])
class PriorCollapseTests(unittest.TestCase):
"""#963: a collapsed head is chance-level and silent unless finetune warns."""
@staticmethod
def rows_of(logits, target, n=8, qtype=None, k=None):
qtype = QTYPES["choice"] if qtype is None else qtype
k = len(logits) if k is None else k
return [(qtype, list(logits), list(target), k) for _ in range(n)]
def test_constant_logits_match_the_prior_and_warn(self):
from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message
# Tiny 4-way set: every row is the same logit, so the predicted distribution is the
# uniform prior. This is the measured failure in #963 (range 0.01, CE at ln 4).
records = self.rows_of([0.002, 0.0, -0.001, 0.001], [1.0, 0.0, 0.0, 0.0])
stats = collapse_stats(records)
self.assertIsNotNone(stats)
self.assertLess(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)
self.assertAlmostEqual(stats["mean_ce"], math.log(4), places=2)
msg = prior_collapse_message(records)
self.assertIsNotNone(msg)
self.assertIn("collapsed to the class prior", msg)
self.assertIn("more epochs", msg)
self.assertIn("more data", msg)
self.assertIn("another seed", msg)
def test_peaked_logits_on_a_learnable_set_do_not_warn(self):
from laya.train import prior_collapse_message
records = self.rows_of([5.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0])
self.assertIsNone(prior_collapse_message(records))
self.assertIsNone(prior_collapse_message([]))
self.assertIsNone(prior_collapse_message(self.rows_of([0.0], [1.0], k=1)))
def test_ce_stuck_at_ln_k_warns_even_when_logit_range_is_not_tiny(self):
from laya.train import COLLAPSE_LOGIT_RANGE, collapse_stats, prior_collapse_message
# Softmax is still near-uniform, so CE sits on ln K, but the within-row range is
# above the constant-logit cutoff -- the other #963 trigger.
records = self.rows_of([0.2, 0.05, -0.05, -0.2], [0.25, 0.25, 0.25, 0.25])
stats = collapse_stats(records)
self.assertGreaterEqual(stats["mean_logit_range"], COLLAPSE_LOGIT_RANGE)
self.assertAlmostEqual(stats["mean_ce"], stats["mean_ln_k"], delta=0.03)
msg = prior_collapse_message(records)
self.assertIsNotNone(msg)
self.assertIn("cross-entropy", msg)
def test_finetune_emits_the_collapse_warning_only_when_logits_collapse(self):
from laya.train import prior_collapse_message
collapsed = self.rows_of([0.01, 0.0, -0.01], [1.0, 0.0, 0.0], n=12)
peaked = self.rows_of([4.0, 0.0, 0.0], [1.0, 0.0, 0.0], n=12)
self.assertIsNotNone(prior_collapse_message(collapsed))
self.assertIsNone(prior_collapse_message(peaked))
def run(records):
with tempfile.TemporaryDirectory() as d:
root = Path(d)
make_checkpoint(root / "base", make_tokenizer())
config = TrainConfig(epochs=1, micro_batch=8, grad_accum=1, calib_frac=0.2, log_every=0)
with patch("huggingface_hub.snapshot_download",
side_effect=AssertionError("unexpected download")), \
patch("laya.train.read_jsonl", return_value=rows(2)), \
patch("laya.train.calibration_records", return_value=records), \
warnings.catch_warnings(record=True) as caught:
warnings.simplefilter("always")
finetune("rows.jsonl", str(root / "base"), str(root / "out"), config, device="cpu")
return [str(w.message) for w in caught if issubclass(w.category, RuntimeWarning)]
collapsed_msgs = run(collapsed)
self.assertTrue(any("collapsed to the class prior" in m for m in collapsed_msgs), collapsed_msgs)
peaked_msgs = run(peaked)
self.assertFalse(any("collapsed to the class prior" in m for m in peaked_msgs), peaked_msgs)
class OptimizerBudgetTests(unittest.TestCase):
"""#963 proposal 3: the optimizer-update budget is derived once and surfaced before a run."""
def test_updates_count_every_accumulation_window_including_the_final_partial_one(self):
from laya.train import optimizer_updates
# steps_per_epoch = ceil(N / micro_batch); one update per full window plus the final
# partial window of every epoch. typed-decisions and the #963 set are the reference points.
self.assertEqual(optimizer_updates(6000, 8, 8, 4), 376)
self.assertEqual(optimizer_updates(1100, 8, 8, 4), 72)
self.assertEqual(optimizer_updates(5, 2, 2, 2), 4) # 3 micro-steps -> 2 updates/epoch
self.assertEqual(optimizer_updates(63, 8, 8, 4), 4) # fewer micro-steps than a window
self.assertEqual(optimizer_updates(8, 8, 8, 1), 1)
self.assertEqual(optimizer_updates(9, 8, 8, 1), 1) # 2 micro-steps -> 1 update
with self.assertRaises(ValueError):
optimizer_updates(0, 8, 8, 4)
def test_train_model_takes_exactly_optimizer_updates_steps(self):
from laya.train import optimizer_updates
class ScalarModel(torch.nn.Module):
def __init__(self):
super().__init__()
self.encoder = torch.nn.Identity()
self.value = torch.nn.Parameter(torch.tensor(0.0))
steps = []
class CountingSGD(torch.optim.SGD):
def step(self, *args, **kwargs):
steps.append(1)
return super().step(*args, **kwargs)
tok = type("Tokenizer", (), {"pad_token_id": 0})()
item = {"q": {"t": "choice"}, "k": 2}
batch = {
"marker_mask": torch.ones((1, 1), dtype=torch.bool),
"target": torch.ones((1, 1)),
}
for n_items, micro_batch, grad_accum, epochs in [(5, 2, 2, 2), (3, 1, 2, 1),
(7, 3, 2, 3), (8, 8, 8, 1)]:
with self.subTest(n_items=n_items, micro_batch=micro_batch, grad_accum=grad_accum):
model = ScalarModel()
steps.clear()
config = TrainConfig(
epochs=epochs, micro_batch=micro_batch, grad_accum=grad_accum,
head_lr=1.0, min_lr=1.0, weight_decay=0.0, grad_clip=100.0,
loss="soft-ce", freeze_encoder=True, amp=False, log_every=0,
)
with patch("laya.train.encode_item", return_value={}), \
patch("laya.train.collate_items", return_value=batch), \
patch("laya.train._forward", side_effect=lambda current, *_args: current.value), \
patch("laya.train.soft_ce_loss", side_effect=lambda logits, *_args: logits), \
patch("laya.train.torch.optim.AdamW",
side_effect=lambda groups, weight_decay: CountingSGD(
groups, weight_decay=weight_decay)):
train_model(model, tok, [item] * n_items, config, torch.device("cpu"), 1, 1)
self.assertEqual(len(steps), optimizer_updates(n_items, micro_batch, grad_accum, epochs))
def test_dry_run_reports_the_post_calibration_budget(self):
import contextlib
import io
from laya.train import dry_run, optimizer_updates
with tempfile.TemporaryDirectory() as d:
root = Path(d)
make_checkpoint(root / "base", make_tokenizer())
config = TrainConfig(epochs=3, micro_batch=4, grad_accum=2, calib_frac=0.25, log_every=0)
buf = io.StringIO()
with patch("laya.train.read_data", return_value=rows(2)), contextlib.redirect_stdout(buf):
summary = dry_run("synthetic.jsonl", str(root / "base"), config)
# The budget must come from the post-calibration split, not the raw valid-item count.
self.assertGreater(summary["calibration_items"], 0)
self.assertEqual(summary["train_items"] + summary["calibration_items"], summary["valid_items"])
self.assertLess(summary["train_items"], summary["valid_items"])
self.assertEqual(summary["optimizer_updates"],
optimizer_updates(summary["train_items"], 4, 2, 3))
self.assertIn("optimizer budget", buf.getvalue())
self.assertIn("%d optimizer updates" % summary["optimizer_updates"], buf.getvalue())
if __name__ == "__main__":
unittest.main()