Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat(train): add laya.train, one fine-tuning loop for the notebook and scripts to share
GuilhermeFdeb477dc8546
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex ec5d2d8c..af943426 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -247,7 +247,7 @@ jobs:test_empty_questions test_router_memory test_shortlist_cosine \test_temperature_loading test_confidence test_cli test_mcp \test_mcp_device \- test_langchain test_portability test_training \+ test_langchain test_portability test_training test_train \test_example_server_limits test_blank_lang_routing \test_export_onnx_safety test_load_errors test_revision_pinning; doif [ -f "tests/$t.py" ]; then python "tests/$t.py"; fidiff --git a/laya/train.py b/laya/train.pynew file mode 100644index 00000000..5e861f85--- /dev/null+++ b/laya/train.py@@ -0,0 +1,502 @@+"""Fine-tune a Laya checkpoint on typed decisions: one training loop the scripts can share.++The notebook, the Apple Silicon script and `research/scripts/finetune_single_device.py` each+carry their own copy of the same loop. This module is that loop once, with two additions the+copies do not have:++- **A choice of objective.** `loss="rlcd"` (the default) is the notebook's objective: a+ GRPO-style term over noisy logit samples rewarded by `proper_reward`, plus soft+ cross-entropy against the target distribution. `loss="soft-ce"` trains on the soft+ cross-entropy alone; #741 measured no gain from the extra term on the typed-decisions split.+- **Option-order augmentation (opt-in).** The scripts always present options in the order the+ question lists them, so the model can learn a position prior: the 20-option order-flip rate in+ `BENCHMARKS.md` and #131 are both that symptom. Each epoch, items of the types named in+ `shuffle_options` (for example `("choice",)`) are re-encoded with a random `option_order` -- the same path inference+ uses -- and the target is permuted with them, so slot `s` is trained on option+ `option_order[s]`.++Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned checkpoint is+fitted with the same clamp and buckets the runtime applies instead of a local copy of the+fitter.++Items keep the question and the tokenized state rather than a finished sequence, because a+shuffled epoch needs to rebuild the head. States are tokenized once per row and shared by+every question on it.++No new dependency: torch, transformers and safetensors are already required by the package.+"""+import json+import math+import os+import random+from dataclasses import asdict, dataclass+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple++import torch++from .calibrate import fit_temperature_map+from .common import (+ QTYPES,+ build_head,+ build_model,+ build_sequence,+ collate_items,+ encode_text,+ proper_reward,+ render_options,+ serialize_state,+)++LOSSES = ("soft-ce", "rlcd")+_MASKED = -1e4+++@dataclass+class TrainConfig:+ """Training knobs. The defaults train the way the published notebook does.++ The notebook's effective batch of 64 is 8 per micro-batch x 2 GPUs x 4 accumulation steps;+ on one device that is `micro_batch=8, grad_accum=8`.++ `max_len` / `head_max_len` default to the checkpoint's own config. Whatever is used is+ written into the saved config, so inference sees the same budgets training did.+ `amp` defaults to on for CUDA only; `gradient_checkpointing` defaults to following `amp`.+ """++ epochs: int = 4+ micro_batch: int = 8+ grad_accum: int = 8+ encoder_lr: float = 2.5e-5+ head_lr: float = 1e-4+ min_lr: float = 1e-6+ weight_decay: float = 0.01+ grad_clip: float = 1.0+ loss: str = "rlcd"+ rl_samples: int = 4+ sigma_start: float = 0.4+ sigma_end: float = 0.1+ w_sph: float = 0.75+ w_rps: float = 1.0+ shuffle_options: Tuple[str, ...] = ()+ freeze_encoder: bool = False+ calib_max: int = 400+ calib_frac: float = 0.1+ calib_seed: int = 20260922+ seed: int = 0+ max_len: Optional[int] = None+ head_max_len: Optional[int] = None+ amp: Optional[bool] = None+ gradient_checkpointing: Optional[bool] = None+ log_every: int = 100++ def validate(self) -> None:+ if self.loss not in LOSSES:+ raise ValueError("loss must be one of %s, got %r" % (", ".join(LOSSES), self.loss))+ unknown = sorted(set(self.shuffle_options) - set(QTYPES))+ if unknown:+ raise ValueError("shuffle_options names unknown question type(s): %s" % ", ".join(unknown))+ for name in ("epochs", "micro_batch", "grad_accum", "rl_samples"):+ value = getattr(self, name)+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:+ raise ValueError("%s must be a positive integer, got %r" % (name, value))+ if not 0.0 <= self.calib_frac < 1.0:+ raise ValueError("calib_frac must be in [0, 1), got %r" % (self.calib_frac,))+++# ------------------------------------------------------------------------------------- data++def to_internal(qid: Any, question: Dict[str, Any]) -> Dict[str, Any]:+ """The question as inference sees it: `Agent._check_question` then `Agent._to_internal`.++ Training on any other rendering would teach the model option texts it never sees at+ inference -- a list-form choice, custom noul `labels`, a non-string instruction.+ """+ from .agent import Agent++ Agent._check_question(qid, question)+ q = Agent._to_internal(question)+ q.pop("option_order", None) # training draws its own order per epoch+ return q+++def target_from_gold(q: Dict[str, Any], gold_q: Dict[str, Any]) -> List[float]:+ """The normalised target distribution for an internal question (`to_internal`), in option order.++ `gold_q["probabilities"]` is keyed by choice label, by `"false"`/`"true"` for noul, and by+ the level index as a string for score -- the `{state, questions, gold}` schema in+ `docs/finetune.md`. A choice label that is not a string is also looked up as its JSON key.+ A row whose probabilities sum to zero becomes uniform, as in the scripts. A negative or+ non-finite probability raises `ValueError`.+ """+ probs = gold_q["probabilities"]++ def get(key, default):+ return probs.get(key, probs.get(str(key), default))++ t, crit = q["t"], q["crit"]+ if t == "choice":+ target = [float(get(k, 0.0)) for k in crit.keys()]+ elif t == "noul":+ target = [float(get("false", 0.5)), float(get("true", 0.5))]+ else:+ target = [float(get(str(i), 0.0)) for i in range(len(crit))]+ if any(not math.isfinite(v) or v < 0 for v in target):+ raise ValueError("probabilities must be finite and non-negative, got %r" % (target,))+ total = sum(target)+ if total > 0:+ return [v / total for v in target]+ return [1.0 / len(target)] * len(target)+++def encode_state(tok, state: Any, max_len: int) -> List[int]:+ """State token ids, capped at `max_len`: no sequence can hold more of the state than that."""+ text = serialize_state(state).replace(tok.mask_token, " ")+ return encode_text(tok, text, add_special_tokens=False)["input_ids"][:max_len]+++def make_item(tok, q: Dict[str, Any], target: Sequence[float], state_ids: List[int],+ head_max_len: int) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:+ """`(item, None)`, or `(None, reason)` when the question cannot be trained on as written.++ `q` is an internal question (`to_internal`). The reasons are `target_mismatch` (the target+ does not have one entry per option) and `options_collapsed` (the head budget left two options+ with the same token span, #538). Training on either would teach the model to tell apart+ options it cannot see as different.+ """+ k = len(render_options(q))+ if len(target) != k:+ return None, "target_mismatch"+ _ids, markers, stats = build_head(tok, q, head_max_len)+ if len(markers) != k or stats["options_distinct"] < stats["options"]:+ return None, "options_collapsed"+ return {"q": q, "state_ids": state_ids, "target": [float(v) for v in target],+ "qtype": QTYPES[q["t"]], "k": k}, None+++def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,+ head_max_len: int) -> Tuple[List[Dict[str, Any]], Dict[str, int]]:+ """Items from `{state, questions, gold}` rows; returns `(items, skipped)`.++ `skipped` counts, by reason, questions that were labelled but could not become an item:+ `invalid_question` and `invalid_target` (a `ValueError` from `to_internal` or+ `target_from_gold`), plus the two reasons `make_item` gives. Questions with no gold entry+ are not counted -- the row simply does not label them.+ """+ items, skipped = [], {}+ for row in rows:+ state_ids = encode_state(tok, row["state"], max_len)+ for qid, question in row["questions"].items():+ gold_q = row["gold"].get(qid)+ if gold_q is None:+ continue+ reason = None+ try:+ q = to_internal(qid, question)+ except ValueError:+ reason = "invalid_question"+ if reason is None:+ try:+ target = target_from_gold(q, gold_q)+ except (ValueError, TypeError, KeyError):+ reason = "invalid_target"+ if reason is None:+ item, reason = make_item(tok, q, target, state_ids, head_max_len)+ if reason is None:+ items.append(item)+ else:+ skipped[reason] = skipped.get(reason, 0) + 1+ return items, skipped+++def read_jsonl(path: str) -> List[Dict[str, Any]]:+ rows = []+ with open(path, encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if line and not line.startswith("#"):+ rows.append(json.loads(line))+ return rows+++def encode_item(tok, item: Dict[str, Any], max_len: int, head_max_len: int,+ option_order: Optional[List[int]] = None) -> Dict[str, Any]:+ """The model input for an item, with options in `option_order` and the target to match.++ `build_sequence` puts option `option_order[s]` in slot `s`, so the slot-ordered target is+ `target[option_order[s]]` -- the inverse of what `unpermute_probs` does at inference.+ """+ ids, markers = build_sequence(tok, None, item["q"], max_len, head_max_len,+ option_order=option_order, state_ids=item["state_ids"])+ target = item["target"]+ if option_order is not None:+ target = [target[i] for i in option_order]+ return {"ids": ids, "markers": markers, "qtype": item["qtype"], "target": target}+++def draw_option_order(item: Dict[str, Any], rng: random.Random,+ shuffle_types: Sequence[str]) -> Optional[List[int]]:+ """A random permutation for items whose type is in `shuffle_types`, else None (canonical)."""+ if item["q"]["t"] not in shuffle_types or item["k"] < 2:+ return None+ order = list(range(item["k"]))+ rng.shuffle(order)+ return order+++def split_calibration(items: Sequence[Any], calib_max: int, calib_frac: float,+ seed: int) -> Tuple[List[Any], List[Any]]:+ """`(train, calibration)`, with the calibration slice taken before any training.++ Fitting temperatures on items the run has trained on measures the fit, not the calibration+ (`docs/finetune.md`). The split depends only on `seed` and the item count, so every DDP rank+ and every rerun hold out the same items.+ """+ n_calib = min(calib_max, int(len(items) * calib_frac))+ order = list(range(len(items)))+ random.Random(seed).shuffle(order)+ held = set(order[:n_calib])+ train = [it for i, it in enumerate(items) if i not in held]+ calib = [it for i, it in enumerate(items) if i in held]+ return train, calib+++# ----------------------------------------------------------------------------------- losses++def soft_ce_loss(logits: torch.Tensor, target: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:+ """Cross-entropy against a target distribution, over each row's own options only."""+ logp = torch.log_softmax(logits.masked_fill(~mask, _MASKED), -1)+ return -(target * logp * mask).sum(-1).mean()+++def rlcd_loss(logits: torch.Tensor, target: torch.Tensor, mask: torch.Tensor, qtype: torch.Tensor,+ sigma: float, samples: int = 4, w_sph: float = 0.75, w_rps: float = 1.0) -> torch.Tensor:+ """The notebook objective: a policy-gradient term over noisy logits plus soft cross-entropy.++ Zero-mean Gaussian noise (scaled by `sigma`, projected onto each row's options) is added to+ detached logits; each sample is rewarded by `proper_reward` against the target, and the+ normalised advantage weights the Gaussian log-density of the sample.+ """+ k = mask.sum(-1, keepdim=True).float()+ fmask = mask.float()+ eps = torch.randn((samples,) + logits.shape, device=logits.device) * sigma * fmask+ eps = (eps - eps.sum(-1, keepdim=True) / k) * fmask+ z = logits.detach().unsqueeze(0) + eps+ q = torch.softmax(z.masked_fill(~mask, _MASKED), -1)+ with torch.no_grad():+ r = proper_reward(q, target.unsqueeze(0), qtype, fmask, w_sph=w_sph, w_rps=w_rps)+ adv = r - r.mean(0, keepdim=True)+ adv = adv / (adv.std() + 1e-6)+ logp = -(((z - logits.unsqueeze(0)) ** 2) * fmask).sum(-1) / (2 * sigma ** 2)+ return -(adv * logp).mean() + soft_ce_loss(logits, target, mask)+++def sigma_at(epoch: int, epochs: int, start: float, end: float) -> float:+ """Exploration noise for `epoch`, annealed linearly from `start` to `end`."""+ return start + (end - start) * (epoch / max(1, epochs - 1))+++# ------------------------------------------------------------------------------- checkpoint++def resolve_device(device: Optional[str] = "auto") -> torch.device:+ if device not in (None, "auto"):+ return torch.device(device)+ if torch.cuda.is_available():+ return torch.device("cuda", 0)+ if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():+ return torch.device("mps")+ if hasattr(torch, "xpu") and torch.xpu.is_available():+ return torch.device("xpu")+ return torch.device("cpu")+++def load_checkpoint(model_dir: str):+ """`(model, tokenizer, cfg)` from a local checkpoint directory, model on CPU in fp32."""+ from safetensors.torch import load_file+ from transformers import AutoTokenizer++ from .agent import _fix_tokenizer_config++ _fix_tokenizer_config(model_dir)+ with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:+ cfg = json.load(f)+ tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "tokenizer"))+ model = build_model(cfg, encoder_dir=os.path.join(model_dir, "encoder"))+ model.load_state_dict(load_file(os.path.join(model_dir, "model.safetensors")), strict=True)+ return model.float(), tok, cfg+++def save_checkpoint(model, tok, cfg: Dict[str, Any], path: str) -> None:+ """Write the layout `laya.load` reads: weights in fp16, encoder config, tokenizer, config."""+ from safetensors.torch import save_file++ os.makedirs(path, exist_ok=True)+ save_file({k: v.detach().half().cpu().contiguous() for k, v in model.state_dict().items()},+ os.path.join(path, "model.safetensors"))+ model.encoder.config.save_pretrained(os.path.join(path, "encoder"))+ tok.save_pretrained(os.path.join(path, "tokenizer"))+ tmp = os.path.join(path, "rl_agent_config.json.tmp")+ with open(tmp, "w", encoding="utf-8") as f:+ json.dump(cfg, f, indent=2)+ os.replace(tmp, os.path.join(path, "rl_agent_config.json"))+++# ------------------------------------------------------------------------------------ train++def _forward(model, batch, device, amp: bool, detach_encoder: bool):+ args = (batch["input_ids"].to(device), batch["attention_mask"].to(device),+ batch["marker_pos"].to(device), batch["marker_mask"].to(device), batch["qtype"].to(device))+ if amp:+ with torch.autocast(device.type, dtype=torch.float16):+ logits, _act = model(*args, detach_encoder=detach_encoder)+ else:+ logits, _act = model(*args, detach_encoder=detach_encoder)+ return logits.float()+++def train_model(model, tok, items: Sequence[Dict[str, Any]], config: TrainConfig, device: torch.device,+ max_len: int, head_max_len: int,+ on_epoch_end: Optional[Callable[[int, float], None]] = None) -> List[float]:+ """Train `model` in place on `items`; returns the mean loss of each epoch."""+ config.validate()+ if not items:+ raise ValueError("no training items")+ amp = (device.type == "cuda") if config.amp is None else bool(config.amp)+ checkpointing = amp if config.gradient_checkpointing is None else bool(config.gradient_checkpointing)++ if config.freeze_encoder:+ for p in model.encoder.parameters():+ p.requires_grad_(False)+ elif checkpointing and hasattr(model.encoder, "gradient_checkpointing_enable"):+ model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})+ model.head_checkpointing = checkpointing+ model.to(device).train()+ if config.freeze_encoder:+ # A frozen encoder is a fixed feature extractor: dropout in it would only add noise.+ model.encoder.eval()++ groups = [{"params": [p for n, p in model.named_parameters()+ if not n.startswith("encoder.") and p.requires_grad], "lr": config.head_lr}]+ if not config.freeze_encoder:+ groups.insert(0, {"params": [p for n, p in model.named_parameters()+ if n.startswith("encoder.") and p.requires_grad], "lr": config.encoder_lr})+ optimizer = torch.optim.AdamW(groups, weight_decay=config.weight_decay)+ steps_per_epoch = math.ceil(len(items) / config.micro_batch)+ updates = max(1, math.ceil(steps_per_epoch / config.grad_accum) * config.epochs)+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=updates, eta_min=config.min_lr)+ scaler = torch.amp.GradScaler("cuda") if amp and device.type == "cuda" else None++ torch.manual_seed(config.seed)+ order_rng = random.Random(config.seed)+ params = [p for g in groups for p in g["params"]]+ history = []+ for epoch in range(config.epochs):+ epoch_items = list(items)+ random.Random(config.seed + epoch).shuffle(epoch_items)+ sigma = sigma_at(epoch, config.epochs, config.sigma_start, config.sigma_end)+ total, n_steps = 0.0, 0+ optimizer.zero_grad(set_to_none=True)+ for start in range(0, len(epoch_items), config.micro_batch):+ chunk = [encode_item(tok, it, max_len, head_max_len,+ draw_option_order(it, order_rng, config.shuffle_options))+ for it in epoch_items[start:start + config.micro_batch]]+ batch = collate_items([chunk], tok.pad_token_id)+ logits = _forward(model, batch, device, amp, config.freeze_encoder)+ mask = batch["marker_mask"].to(device)+ target = batch["target"].to(device)+ if config.loss == "rlcd":+ loss = rlcd_loss(logits, target, mask, batch["qtype"].to(device), sigma,+ config.rl_samples, config.w_sph, config.w_rps)+ else:+ loss = soft_ce_loss(logits, target, mask)+ scaled = loss / config.grad_accum+ if scaler is not None:+ scaler.scale(scaled).backward()+ else:+ scaled.backward()+ n_steps += 1+ if n_steps % config.grad_accum == 0 or start + config.micro_batch >= len(epoch_items):+ if scaler is not None:+ scaler.unscale_(optimizer)+ torch.nn.utils.clip_grad_norm_(params, config.grad_clip)+ if scaler is not None:+ scaler.step(optimizer)+ scaler.update()+ else:+ optimizer.step()+ scheduler.step()+ optimizer.zero_grad(set_to_none=True)+ total += loss.item()+ if config.log_every and n_steps % config.log_every == 0:+ print("epoch %d/%d step %d loss %.4f" % (epoch + 1, config.epochs, n_steps, loss.item()), flush=True)+ mean = total / max(1, n_steps)+ history.append(mean)+ print("epoch %d/%d mean loss %.4f" % (epoch + 1, config.epochs, mean), flush=True)+ if on_epoch_end is not None:+ on_epoch_end(epoch, mean)+ model.eval()+ return history++[email protected]_grad()+def calibration_records(model, tok, items: Sequence[Dict[str, Any]], device: torch.device,+ max_len: int, head_max_len: int, batch_size: int = 16):+ """`(qtype, logits, target, k)` records for `fit_temperature_map`, options in canonical order."""+ model.eval()+ records = []+ for start in range(0, len(items), batch_size):+ chunk = items[start:start + batch_size]+ batch = collate_items([[encode_item(tok, it, max_len, head_max_len) for it in chunk]], tok.pad_token_id)+ logits = _forward(model, batch, device, amp=False, detach_encoder=False).cpu().numpy()+ for row, it in zip(logits, chunk):+ records.append((it["qtype"], row[:it["k"]], it["target"], it["k"]))+ return records+++def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,+ device: Optional[str] = "auto") -> Dict[str, Any]:+ """Preprocess, train, calibrate and save; returns a summary of the run.++ `data` is a JSONL file of `{state, questions, gold}` rows (the schema in `docs/finetune.md`).+ `model_dir` is a local checkpoint directory -- the layout `laya.load` reads. The result in+ `output_dir` loads with `laya.load(output_dir)`.+ """+ config = config or TrainConfig()+ config.validate()+ dev = resolve_device(device)+ model, tok, cfg = load_checkpoint(model_dir)+ max_len = config.max_len or cfg.get("max_len", 512)+ head_max_len = config.head_max_len or cfg.get("head_max_len", 192)++ items, skipped = items_from_rows(tok, read_jsonl(data), max_len, head_max_len)+ if not items:+ raise ValueError("%s produced no training items (skipped: %r)" % (data, skipped))+ train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)+ print("train items %d, calibration items %d, skipped %r, device %s"+ % (len(train_items), len(calib_items), skipped, dev), flush=True)++ def checkpoint_latest(epoch, _loss):+ save_checkpoint(model, tok, dict(cfg, max_len=max_len, head_max_len=head_max_len),+ os.path.join(output_dir, "checkpoint_latest"))++ history = train_model(model, tok, train_items, config, dev, max_len, head_max_len,+ on_epoch_end=checkpoint_latest)++ fitted = fit_temperature_map(calibration_records(model, tok, calib_items, dev, max_len, head_max_len))+ out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,+ temperature=fitted["temperature"])+ # An inherited bucket map takes precedence at inference and would mask the new fit.+ out_cfg.pop("temperature_by_options", None)+ if fitted["temperature_by_options"]:+ out_cfg["temperature_by_options"] = fitted["temperature_by_options"]+ out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))+ save_checkpoint(model, tok, out_cfg, output_dir)+ return {+ "train_items": len(train_items),+ "calibration_items": len(calib_items),+ "skipped": skipped,+ "epoch_loss": history,+ "temperature": fitted["temperature"],+ "temperature_by_options": fitted["temperature_by_options"],+ "n_by_bucket": fitted["n_by_bucket"],+ "output_dir": output_dir,+ }diff --git a/scripts/test_suites.py b/scripts/test_suites.pyindex ffe1ab20..99578629 100644--- a/scripts/test_suites.py+++ b/scripts/test_suites.py@@ -84,6 +84,7 @@ SCRIPT_SUITES = ["tests/test_langchain.py","tests/test_portability.py","tests/test_training.py",+ "tests/test_train.py","tests/test_example_server_limits.py","tests/test_blank_lang_routing.py","tests/test_export_onnx_safety.py",diff --git a/tests/test_train.py b/tests/test_train.pynew file mode 100644index 00000000..f7ca5558--- /dev/null+++ b/tests/test_train.py@@ -0,0 +1,396 @@+"""laya.train: data prep, option-order augmentation, losses and an end-to-end run.++CPU only and weight-free: the checkpoint is a one-layer BERT with a word-level tokenizer built+in a temp directory, so nothing is downloaded.++Run: python tests/test_train.py+"""+import json+import os+from pathlib import Path+import random+import sys+import tempfile+import unittest+from unittest.mock import patch++os.environ.setdefault("USE_TF", "0")+os.environ.setdefault("USE_TORCH", "1")+os.environ.setdefault("HF_HUB_OFFLINE", "1")+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))++import numpy as np # noqa: E402+import torch # noqa: E402+from safetensors.torch import load_file, save_file # noqa: E402+from tokenizers import Tokenizer # noqa: E402+from tokenizers.models import WordLevel # noqa: E402+from tokenizers.pre_tokenizers import Whitespace # noqa: E402+from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402++from laya import load # noqa: E402+from laya.common import ( # noqa: E402+ DecisionModel,+ QTYPES,+ TEMP_MAX,+ TEMP_MIN,+ build_sequence,+ proper_reward,+ render_options,+ unpermute_probs,+)+from laya.train import ( # noqa: E402+ TrainConfig,+ draw_option_order,+ encode_item,+ encode_state,+ finetune,+ items_from_rows,+ make_item,+ rlcd_loss,+ soft_ce_loss,+ split_calibration,+ target_from_gold,+ to_internal,+ train_model,+)++WORDS = ["refund", "invoice", "charged", "twice", "crash", "error", "app", "screen", "please",+ "billing", "technical", "other", "is", "it", "urgent", "low", "high", "level", "question",+ "choice", "score", "noul", "which", "team", "yes", "no", "the", "statement", "does",+ "not", "hold", "holds", "one", "two", "three", "four", "five", "six", "seven", ":", ",", "?"]++DEPARTMENT = {"type": "choice", "instructions": "which team ?",+ "criteria": {"billing": "", "technical": "", "other": ""}}+URGENT = {"type": "noul", "instructions": "is it urgent ?"}+LEVEL = {"type": "score", "instructions": "which level ?", "criteria": ["low", "high"]}++BILLING_STATES = ["please refund invoice", "charged twice invoice", "refund charged twice", "invoice refund please"]+TECH_STATES = ["app crash error", "screen error crash", "crash app screen", "error screen app"]+++def make_tokenizer():+ vocab = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}+ for w in WORDS:+ vocab[w] = len(vocab)+ backend = Tokenizer(WordLevel(vocab, unk_token="[UNK]"))+ backend.pre_tokenizer = Whitespace()+ return PreTrainedTokenizerFast(tokenizer_object=backend, pad_token="[PAD]", unk_token="[UNK]",+ cls_token="[CLS]", sep_token="[SEP]", mask_token="[MASK]")+++def make_checkpoint(path: Path, tok):+ torch.manual_seed(0)+ config = BertConfig(vocab_size=len(tok), hidden_size=32, num_hidden_layers=1,+ num_attention_heads=2, intermediate_size=64, max_position_embeddings=128)+ config.save_pretrained(path / "encoder")+ tok.save_pretrained(path / "tokenizer")+ model = DecisionModel(BertModel(config), head_layers=1, n_act=2)+ save_file(model.state_dict(), path / "model.safetensors")+ cfg = {"encoder": "unused/offline", "head_layers": 1, "act_costs": {"act": 0},+ "max_len": 64, "head_max_len": 40, "temperature": [1.0, 1.0, 1.0],+ "temperature_by_options": {"choice:3-5": 0.7}}+ (path / "rl_agent_config.json").write_text(json.dumps(cfg))+++def rows(n_repeats=6):+ out = []+ for _ in range(n_repeats):+ for s in BILLING_STATES:+ out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},+ "gold": {"department": {"probabilities": {"billing": 0.9, "technical": 0.05, "other": 0.05}},+ "urgent": {"probabilities": {"false": 0.8, "true": 0.2}},+ "level": {"probabilities": {"0": 0.7, "1": 0.3}}}})+ for s in TECH_STATES:+ out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},+ "gold": {"department": {"probabilities": {"billing": 0.05, "technical": 0.9, "other": 0.05}},+ "urgent": {"probabilities": {"false": 0.2, "true": 0.8}},+ "level": {"probabilities": {"0": 0.2, "1": 0.8}}}})+ return out+++class DataTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.tok = make_tokenizer()++ def item(self, question, target, state="app crash", head_max_len=40):+ item, reason = make_item(self.tok, to_internal("q", question), target,+ encode_state(self.tok, state, 64), head_max_len)+ self.assertIsNone(reason)+ return item++ def test_target_from_gold_orders_and_normalises(self):+ def target(question, probs):+ return target_from_gold(to_internal("q", question), {"probabilities": probs})++ self.assertEqual(target(DEPARTMENT, {"technical": 2.0, "billing": 2.0}), [0.5, 0.5, 0.0])+ self.assertEqual(target(URGENT, {"true": 0.75, "false": 0.25}), [0.25, 0.75])+ self.assertEqual(target(LEVEL, {"1": 1.0}), [0.0, 1.0])+ self.assertEqual(target(DEPARTMENT, {}), [1 / 3] * 3)+ # list-form choice and non-string labels, which inference accepts+ self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},+ {"screen": 1.0}), [0.0, 1.0])+ self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": {1: "", 2: ""}},+ {"2": 3.0, "1": 1.0}), [0.25, 0.75])+ for bad in ({"billing": -1.0}, {"billing": float("nan")}, {"billing": "high"}):+ with self.assertRaises(ValueError):+ target(DEPARTMENT, bad)++ def test_questions_are_rendered_exactly_as_inference_renders_them(self):+ # The sequence a training item produces must be the one `Agent` builds for the same+ # question, or the model is trained on option texts it never sees.+ from laya.agent import Agent++ questions = [+ DEPARTMENT, URGENT, LEVEL,+ {"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},+ {"type": "noul", "instructions": "is it ?", "labels": {"false": "no", "true": "yes"},+ "criteria": {True: "holds", False: "does not hold"}},+ {"type": "noul", "instructions": {"which": "team"}},+ ]+ for question in questions:+ with self.subTest(question=question):+ k = len(render_options(Agent._to_internal(question)))+ item = self.item(question, [1.0 / k] * k)+ expected = build_sequence(self.tok, "app crash", Agent._to_internal(question), 64, 40)+ enc = encode_item(self.tok, item, 64, 40)+ self.assertEqual((enc["ids"], enc["markers"]), expected)++ def test_rows_share_one_tokenized_state(self):+ items, skipped = items_from_rows(self.tok, rows(1)[:1], 64, 40)+ self.assertEqual((len(items), skipped), (3, {}))+ self.assertTrue(items[0]["state_ids"] is items[1]["state_ids"] is items[2]["state_ids"])+ self.assertEqual([it["qtype"] for it in items], [QTYPES["choice"], QTYPES["noul"], QTYPES["score"]])++ def test_question_without_gold_is_not_an_item_or_a_skip(self):+ row = dict(rows(1)[0])+ row["gold"] = {"department": row["gold"]["department"]}+ items, skipped = items_from_rows(self.tok, [row], 64, 40)+ self.assertEqual((len(items), skipped), (1, {}))++ def test_bad_rows_are_skipped_with_a_reason(self):+ row = dict(rows(1)[0])+ row["questions"] = dict(row["questions"],+ broken={"type": "choice", "instructions": "which ?", "criteria": []},+ negative=DEPARTMENT)+ row["gold"] = dict(row["gold"], broken={"probabilities": {}},+ negative={"probabilities": {"billing": -2.0}})+ items, skipped = items_from_rows(self.tok, [row], 64, 40)+ self.assertEqual((len(items), skipped), (3, {"invalid_question": 1, "invalid_target": 1}))++ def test_options_the_head_budget_collapses_are_skipped(self):+ # Two options that only differ after the per-option cap come out as the same span (#538).+ q = to_internal("q", {"type": "choice", "instructions": "which ?",+ "criteria": {"one two three four five six": "", "one two three four five seven": ""}})+ state_ids = encode_state(self.tok, "app", 64)+ self.assertEqual(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=20),+ (None, "options_collapsed"))+ self.assertIsNotNone(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=64)[0])++ def test_target_length_must_match_options(self):+ state_ids = encode_state(self.tok, "app", 64)+ self.assertEqual(make_item(self.tok, to_internal("q", DEPARTMENT), [0.5, 0.5], state_ids, 40),+ (None, "target_mismatch"))++ def test_canonical_encoding_matches_build_sequence(self):+ item = self.item(DEPARTMENT, [0.8, 0.1, 0.1])+ enc = encode_item(self.tok, item, 64, 40)+ ids, markers = build_sequence(self.tok, "app crash", item["q"], 64, 40)+ self.assertEqual((enc["ids"], enc["markers"], enc["target"]), (ids, markers, [0.8, 0.1, 0.1]))++ def test_permuted_item_carries_a_permuted_target(self):+ item = self.item(DEPARTMENT, [0.7, 0.2, 0.1])+ order = [2, 0, 1]+ enc = encode_item(self.tok, item, 64, 40, option_order=order)+ self.assertEqual(enc["target"], [0.1, 0.7, 0.2])+ labels = list(DEPARTMENT["criteria"])+ # The token right after slot s's [MASK] is the label of option order[s].+ for slot, marker in enumerate(enc["markers"]):+ self.assertEqual(enc["ids"][marker], self.tok.mask_token_id)+ self.assertEqual(enc["ids"][marker + 1], self.tok.convert_tokens_to_ids(labels[order[slot]]))++ def test_permuted_training_input_is_what_inference_scores_for_that_order(self):+ # Inference with option_order builds the same tokens in the same slots, and+ # unpermute_probs maps the slot-ordered target back to the canonical one.+ from laya.agent import Agent++ order = [1, 2, 0]+ item = self.item(DEPARTMENT, [0.6, 0.3, 0.1])+ enc = encode_item(self.tok, item, 64, 40, option_order=order)+ internal = Agent._to_internal(dict(DEPARTMENT, option_order=order))+ expected_ids, _ = build_sequence(self.tok, "app crash", internal, 64, 40,+ option_order=internal["option_order"])+ self.assertEqual(enc["ids"], expected_ids)+ self.assertEqual(list(unpermute_probs(np.array(enc["target"]), order)), item["target"])++ def test_only_configured_types_are_shuffled(self):+ state_ids = encode_state(self.tok, "app", 64)+ choice, _ = make_item(self.tok, to_internal("q", DEPARTMENT), [1.0, 0.0, 0.0], state_ids, 40)+ noul, _ = make_item(self.tok, to_internal("q", URGENT), [0.5, 0.5], state_ids, 40)+ self.assertIsNone(draw_option_order(noul, random.Random(0), ("choice",)))+ orders = {tuple(draw_option_order(choice, random.Random(s), ("choice",))) for s in range(20)}+ self.assertTrue(all(sorted(o) == [0, 1, 2] for o in orders))+ self.assertGreater(len(orders), 1)+ self.assertEqual(draw_option_order(choice, random.Random(3), ("choice",)),+ draw_option_order(choice, random.Random(3), ("choice",)))+ self.assertIsNone(draw_option_order(choice, random.Random(0), ()))++ def test_calibration_split_is_disjoint_and_seeded(self):+ items = list(range(50))+ train, calib = split_calibration(items, calib_max=400, calib_frac=0.1, seed=7)+ self.assertEqual((len(train), len(calib)), (45, 5))+ self.assertEqual(sorted(train + calib), items)+ self.assertEqual(split_calibration(items, 400, 0.1, 7), (train, calib))+ self.assertEqual(len(split_calibration(items, 3, 0.5, 7)[1]), 3)+++class LossTests(unittest.TestCase):+ def test_soft_ce_ignores_padded_options(self):+ logits = torch.tensor([[2.0, 0.5, 9.0], [1.0, -1.0, 0.0]])+ target = torch.tensor([[0.75, 0.25, 0.0], [0.2, 0.3, 0.5]])+ mask = torch.tensor([[True, True, False], [True, True, True]])+ manual = -(torch.tensor([0.75, 0.25]) * torch.log_softmax(torch.tensor([2.0, 0.5]), -1)).sum()+ manual = (manual - (target[1] * torch.log_softmax(logits[1], -1)).sum()) / 2+ self.assertAlmostEqual(soft_ce_loss(logits, target, mask).item(), manual.item(), places=5)++ def test_rlcd_is_finite_and_differentiable(self):+ torch.manual_seed(0)+ logits = torch.randn(4, 3, requires_grad=True)+ target = torch.softmax(torch.randn(4, 3), -1)+ mask = torch.ones(4, 3, dtype=torch.bool)+ loss = rlcd_loss(logits, target, mask, torch.tensor([0, 1, 2, 0]), sigma=0.4)+ loss.backward()+ self.assertTrue(torch.isfinite(loss))+ self.assertTrue(torch.isfinite(logits.grad).all())++ def test_rlcd_matches_the_notebook_objective(self):+ # The loss as written in research/scripts/finetune_single_device.py, inlined: with the+ # same RNG state, rlcd_loss must give the same value and the same gradient.+ def script_loss(logits, target, mask, qtype, sigma):+ k = mask.sum(-1, keepdim=True).float()+ eps = torch.randn((4,) + logits.shape) * sigma * mask+ eps = (eps - eps.sum(-1, keepdim=True) / k) * mask+ z = logits.detach().unsqueeze(0) + eps+ q = torch.softmax(z.masked_fill(~mask, -1e4), -1)+ with torch.no_grad():+ r = proper_reward(q, target.unsqueeze(0), qtype, mask, w_sph=0.75, w_rps=1.0)+ adv = r - r.mean(0, keepdim=True)+ adv = adv / (adv.std() + 1e-6)+ logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)+ ce = (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()+ return -(adv * logp).mean() - ce++ torch.manual_seed(1)+ base = torch.randn(5, 4)+ mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 0, 0]],+ dtype=torch.bool)+ target = torch.softmax(torch.randn(5, 4), -1) * mask+ target = target / target.sum(-1, keepdim=True)+ qtype = torch.tensor([0, 2, 1, 0, 2])+ results = []+ for fn in (lambda lg: script_loss(lg, target, mask, qtype, 0.3),+ lambda lg: rlcd_loss(lg, target, mask, qtype, 0.3)):+ logits = base.masked_fill(~mask, -1e4).clone().requires_grad_(True)+ torch.manual_seed(5)+ loss = fn(logits)+ loss.backward()+ results.append((loss.item(), logits.grad.clone()))+ self.assertAlmostEqual(results[0][0], results[1][0], places=5)+ self.assertTrue(torch.allclose(results[0][1], results[1][1], atol=1e-6))++ def test_defaults_train_like_the_notebook(self):+ config = TrainConfig()+ self.assertEqual((config.loss, config.shuffle_options), ("rlcd", ()))+ self.assertEqual(config.micro_batch * config.grad_accum, 64)++ def test_config_rejects_unknown_values(self):+ for bad in (TrainConfig(loss="mse"), TrainConfig(shuffle_options=("choise",)),+ TrainConfig(epochs=0), TrainConfig(calib_frac=1.0)):+ with self.assertRaises(ValueError):+ bad.validate()+++class EndToEndTests(unittest.TestCase):+ @classmethod+ def setUpClass(cls):+ cls.tmp = tempfile.TemporaryDirectory()+ cls.addClassCleanup(cls.tmp.cleanup)+ cls.root = Path(cls.tmp.name)+ cls.tok = make_tokenizer()+ make_checkpoint(cls.root / "base", cls.tok)+ cls.data = cls.root / "train.jsonl"+ cls.data.write_text("\n".join(json.dumps(r) for r in rows()), encoding="utf-8")++ def run_finetune(self, out, **overrides):+ # Options stay in order here: a randomly initialised one-layer encoder cannot learn to read+ # option labels in a handful of epochs, only their slots. The permutation itself is+ # covered by DataTests; test_shuffled_run_saves_a_loadable_checkpoint runs it end to end.+ kwargs = dict(epochs=12, micro_batch=8, grad_accum=1, encoder_lr=1e-3, head_lr=1e-3,+ calib_frac=0.2, log_every=0, loss="soft-ce", shuffle_options=())+ kwargs.update(overrides)+ config = TrainConfig(**kwargs)+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):+ return finetune(str(self.data), str(self.root / "base"), str(self.root / out), config, device="cpu")++ def test_finetune_learns_and_saves_a_loadable_checkpoint(self):+ report = self.run_finetune("out")+ self.assertEqual(report["train_items"] + report["calibration_items"], len(rows()) * 3)+ self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])++ saved = json.loads((self.root / "out" / "rl_agent_config.json").read_text())+ self.assertTrue(saved["fine_tuned"])+ self.assertEqual((saved["max_len"], saved["head_max_len"]), (64, 40))+ # The inherited bucket would override the new fit at inference; too few items to refit it.+ self.assertNotIn("temperature_by_options", saved)+ self.assertTrue(all(TEMP_MIN <= t <= TEMP_MAX for t in saved["temperature"]))+ self.assertEqual(saved["training"]["laya_train"]["loss"], "soft-ce")+ self.assertTrue((self.root / "out" / "checkpoint_latest" / "model.safetensors").exists())++ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):+ agent = load(str(self.root / "out"), device="cpu")+ answers = agent.predict("please refund invoice", {"department": DEPARTMENT})["answers"]+ self.assertEqual(answers["department"]["choice"], "billing")+ answers = agent.predict("app crash error", {"department": DEPARTMENT})["answers"]+ self.assertEqual(answers["department"]["choice"], "technical")++ def test_shuffled_run_saves_a_loadable_checkpoint(self):+ report = self.run_finetune("out_shuffled", shuffle_options=("choice",), epochs=1)+ saved = json.loads((self.root / "out_shuffled" / "rl_agent_config.json").read_text())+ self.assertEqual(saved["training"]["laya_train"]["shuffle_options"], ["choice"])+ self.assertEqual(len(report["epoch_loss"]), 1)+ agent = load(str(self.root / "out_shuffled"), device="cpu")+ self.assertIn(agent.predict("app", {"department": DEPARTMENT})["answers"]["department"]["choice"],+ DEPARTMENT["criteria"])++ def test_rlcd_objective_still_trains(self):+ report = self.run_finetune("out_rlcd", loss="rlcd")+ self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])++ def test_frozen_encoder_is_left_untouched(self):+ self.run_finetune("out_frozen", freeze_encoder=True, epochs=2)+ before = load_file(str(self.root / "base" / "model.safetensors"))+ after = load_file(str(self.root / "out_frozen" / "model.safetensors"))+ for name in before:+ if name.startswith("encoder."):+ self.assertTrue(torch.allclose(before[name].half().float(), after[name].float()), name)+ self.assertFalse(torch.allclose(before["scorer.1.weight"].half().float(), after["scorer.1.weight"].float()))++ def test_frozen_encoder_runs_without_dropout(self):+ from laya.train import load_checkpoint++ model, tok, _cfg = load_checkpoint(str(self.root / "base"))+ modes = []+ model.encoder.register_forward_pre_hook(lambda module, _args: modes.append(module.training))+ items, _ = items_from_rows(tok, rows(1)[:4], 64, 40)+ train_model(model, tok, items, TrainConfig(epochs=1, freeze_encoder=True, log_every=0),+ torch.device("cpu"), 64, 40)+ self.assertTrue(modes)+ self.assertFalse(any(modes))++ def test_train_model_rejects_empty_input(self):+ with self.assertRaises(ValueError):+ train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)+++if __name__ == "__main__":+ unittest.main()