Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(train): add laya.train, one fine-tuning loop for the notebook and scripts to share

GuilhermeFdeb477dc8546
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ec5d2d8c..af943426 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -247,7 +247,7 @@ jobs:
test_empty_questions test_router_memory test_shortlist_cosine \
test_temperature_loading test_confidence test_cli test_mcp \
test_mcp_device \
- test_langchain test_portability test_training \
+ test_langchain test_portability test_training test_train \
test_example_server_limits test_blank_lang_routing \
test_export_onnx_safety test_load_errors test_revision_pinning; do
if [ -f "tests/$t.py" ]; then python "tests/$t.py"; fi
diff --git a/laya/train.py b/laya/train.py
new file mode 100644
index 00000000..5e861f85
--- /dev/null
+++ b/laya/train.py
@@ -0,0 +1,502 @@
+"""Fine-tune a Laya checkpoint on typed decisions: one training loop the scripts can share.
+
+The notebook, the Apple Silicon script and `research/scripts/finetune_single_device.py` each
+carry their own copy of the same loop. This module is that loop once, with two additions the
+copies do not have:
+
+- **A choice of objective.** `loss="rlcd"` (the default) is the notebook's objective: a
+ GRPO-style term over noisy logit samples rewarded by `proper_reward`, plus soft
+ cross-entropy against the target distribution. `loss="soft-ce"` trains on the soft
+ cross-entropy alone; #741 measured no gain from the extra term on the typed-decisions split.
+- **Option-order augmentation (opt-in).** The scripts always present options in the order the
+ question lists them, so the model can learn a position prior: the 20-option order-flip rate in
+ `BENCHMARKS.md` and #131 are both that symptom. Each epoch, items of the types named in
+ `shuffle_options` (for example `("choice",)`) are re-encoded with a random `option_order` -- the same path inference
+ uses -- and the target is permuted with them, so slot `s` is trained on option
+ `option_order[s]`.
+
+Calibration goes through `laya.calibrate.fit_temperature_map`, so a fine-tuned checkpoint is
+fitted with the same clamp and buckets the runtime applies instead of a local copy of the
+fitter.
+
+Items keep the question and the tokenized state rather than a finished sequence, because a
+shuffled epoch needs to rebuild the head. States are tokenized once per row and shared by
+every question on it.
+
+No new dependency: torch, transformers and safetensors are already required by the package.
+"""
+import json
+import math
+import os
+import random
+from dataclasses import asdict, dataclass
+from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple
+
+import torch
+
+from .calibrate import fit_temperature_map
+from .common import (
+ QTYPES,
+ build_head,
+ build_model,
+ build_sequence,
+ collate_items,
+ encode_text,
+ proper_reward,
+ render_options,
+ serialize_state,
+)
+
+LOSSES = ("soft-ce", "rlcd")
+_MASKED = -1e4
+
+
+@dataclass
+class TrainConfig:
+ """Training knobs. The defaults train the way the published notebook does.
+
+ The notebook's effective batch of 64 is 8 per micro-batch x 2 GPUs x 4 accumulation steps;
+ on one device that is `micro_batch=8, grad_accum=8`.
+
+ `max_len` / `head_max_len` default to the checkpoint's own config. Whatever is used is
+ written into the saved config, so inference sees the same budgets training did.
+ `amp` defaults to on for CUDA only; `gradient_checkpointing` defaults to following `amp`.
+ """
+
+ epochs: int = 4
+ micro_batch: int = 8
+ grad_accum: int = 8
+ encoder_lr: float = 2.5e-5
+ head_lr: float = 1e-4
+ min_lr: float = 1e-6
+ weight_decay: float = 0.01
+ grad_clip: float = 1.0
+ loss: str = "rlcd"
+ rl_samples: int = 4
+ sigma_start: float = 0.4
+ sigma_end: float = 0.1
+ w_sph: float = 0.75
+ w_rps: float = 1.0
+ shuffle_options: Tuple[str, ...] = ()
+ freeze_encoder: bool = False
+ calib_max: int = 400
+ calib_frac: float = 0.1
+ calib_seed: int = 20260922
+ seed: int = 0
+ max_len: Optional[int] = None
+ head_max_len: Optional[int] = None
+ amp: Optional[bool] = None
+ gradient_checkpointing: Optional[bool] = None
+ log_every: int = 100
+
+ def validate(self) -> None:
+ if self.loss not in LOSSES:
+ raise ValueError("loss must be one of %s, got %r" % (", ".join(LOSSES), self.loss))
+ unknown = sorted(set(self.shuffle_options) - set(QTYPES))
+ if unknown:
+ raise ValueError("shuffle_options names unknown question type(s): %s" % ", ".join(unknown))
+ for name in ("epochs", "micro_batch", "grad_accum", "rl_samples"):
+ value = getattr(self, name)
+ if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+ raise ValueError("%s must be a positive integer, got %r" % (name, value))
+ if not 0.0 <= self.calib_frac < 1.0:
+ raise ValueError("calib_frac must be in [0, 1), got %r" % (self.calib_frac,))
+
+
+# ------------------------------------------------------------------------------------- data
+
+def to_internal(qid: Any, question: Dict[str, Any]) -> Dict[str, Any]:
+ """The question as inference sees it: `Agent._check_question` then `Agent._to_internal`.
+
+ Training on any other rendering would teach the model option texts it never sees at
+ inference -- a list-form choice, custom noul `labels`, a non-string instruction.
+ """
+ from .agent import Agent
+
+ Agent._check_question(qid, question)
+ q = Agent._to_internal(question)
+ q.pop("option_order", None) # training draws its own order per epoch
+ return q
+
+
+def target_from_gold(q: Dict[str, Any], gold_q: Dict[str, Any]) -> List[float]:
+ """The normalised target distribution for an internal question (`to_internal`), in option order.
+
+ `gold_q["probabilities"]` is keyed by choice label, by `"false"`/`"true"` for noul, and by
+ the level index as a string for score -- the `{state, questions, gold}` schema in
+ `docs/finetune.md`. A choice label that is not a string is also looked up as its JSON key.
+ A row whose probabilities sum to zero becomes uniform, as in the scripts. A negative or
+ non-finite probability raises `ValueError`.
+ """
+ probs = gold_q["probabilities"]
+
+ def get(key, default):
+ return probs.get(key, probs.get(str(key), default))
+
+ t, crit = q["t"], q["crit"]
+ if t == "choice":
+ target = [float(get(k, 0.0)) for k in crit.keys()]
+ elif t == "noul":
+ target = [float(get("false", 0.5)), float(get("true", 0.5))]
+ else:
+ target = [float(get(str(i), 0.0)) for i in range(len(crit))]
+ if any(not math.isfinite(v) or v < 0 for v in target):
+ raise ValueError("probabilities must be finite and non-negative, got %r" % (target,))
+ total = sum(target)
+ if total > 0:
+ return [v / total for v in target]
+ return [1.0 / len(target)] * len(target)
+
+
+def encode_state(tok, state: Any, max_len: int) -> List[int]:
+ """State token ids, capped at `max_len`: no sequence can hold more of the state than that."""
+ text = serialize_state(state).replace(tok.mask_token, " ")
+ return encode_text(tok, text, add_special_tokens=False)["input_ids"][:max_len]
+
+
+def make_item(tok, q: Dict[str, Any], target: Sequence[float], state_ids: List[int],
+ head_max_len: int) -> Tuple[Optional[Dict[str, Any]], Optional[str]]:
+ """`(item, None)`, or `(None, reason)` when the question cannot be trained on as written.
+
+ `q` is an internal question (`to_internal`). The reasons are `target_mismatch` (the target
+ does not have one entry per option) and `options_collapsed` (the head budget left two options
+ with the same token span, #538). Training on either would teach the model to tell apart
+ options it cannot see as different.
+ """
+ k = len(render_options(q))
+ if len(target) != k:
+ return None, "target_mismatch"
+ _ids, markers, stats = build_head(tok, q, head_max_len)
+ if len(markers) != k or stats["options_distinct"] < stats["options"]:
+ return None, "options_collapsed"
+ return {"q": q, "state_ids": state_ids, "target": [float(v) for v in target],
+ "qtype": QTYPES[q["t"]], "k": k}, None
+
+
+def items_from_rows(tok, rows: Iterable[Dict[str, Any]], max_len: int,
+ head_max_len: int) -> Tuple[List[Dict[str, Any]], Dict[str, int]]:
+ """Items from `{state, questions, gold}` rows; returns `(items, skipped)`.
+
+ `skipped` counts, by reason, questions that were labelled but could not become an item:
+ `invalid_question` and `invalid_target` (a `ValueError` from `to_internal` or
+ `target_from_gold`), plus the two reasons `make_item` gives. Questions with no gold entry
+ are not counted -- the row simply does not label them.
+ """
+ items, skipped = [], {}
+ for row in rows:
+ state_ids = encode_state(tok, row["state"], max_len)
+ for qid, question in row["questions"].items():
+ gold_q = row["gold"].get(qid)
+ if gold_q is None:
+ continue
+ reason = None
+ try:
+ q = to_internal(qid, question)
+ except ValueError:
+ reason = "invalid_question"
+ if reason is None:
+ try:
+ target = target_from_gold(q, gold_q)
+ except (ValueError, TypeError, KeyError):
+ reason = "invalid_target"
+ if reason is None:
+ item, reason = make_item(tok, q, target, state_ids, head_max_len)
+ if reason is None:
+ items.append(item)
+ else:
+ skipped[reason] = skipped.get(reason, 0) + 1
+ return items, skipped
+
+
+def read_jsonl(path: str) -> List[Dict[str, Any]]:
+ rows = []
+ with open(path, encoding="utf-8") as f:
+ for line in f:
+ line = line.strip()
+ if line and not line.startswith("#"):
+ rows.append(json.loads(line))
+ return rows
+
+
+def encode_item(tok, item: Dict[str, Any], max_len: int, head_max_len: int,
+ option_order: Optional[List[int]] = None) -> Dict[str, Any]:
+ """The model input for an item, with options in `option_order` and the target to match.
+
+ `build_sequence` puts option `option_order[s]` in slot `s`, so the slot-ordered target is
+ `target[option_order[s]]` -- the inverse of what `unpermute_probs` does at inference.
+ """
+ ids, markers = build_sequence(tok, None, item["q"], max_len, head_max_len,
+ option_order=option_order, state_ids=item["state_ids"])
+ target = item["target"]
+ if option_order is not None:
+ target = [target[i] for i in option_order]
+ return {"ids": ids, "markers": markers, "qtype": item["qtype"], "target": target}
+
+
+def draw_option_order(item: Dict[str, Any], rng: random.Random,
+ shuffle_types: Sequence[str]) -> Optional[List[int]]:
+ """A random permutation for items whose type is in `shuffle_types`, else None (canonical)."""
+ if item["q"]["t"] not in shuffle_types or item["k"] < 2:
+ return None
+ order = list(range(item["k"]))
+ rng.shuffle(order)
+ return order
+
+
+def split_calibration(items: Sequence[Any], calib_max: int, calib_frac: float,
+ seed: int) -> Tuple[List[Any], List[Any]]:
+ """`(train, calibration)`, with the calibration slice taken before any training.
+
+ Fitting temperatures on items the run has trained on measures the fit, not the calibration
+ (`docs/finetune.md`). The split depends only on `seed` and the item count, so every DDP rank
+ and every rerun hold out the same items.
+ """
+ n_calib = min(calib_max, int(len(items) * calib_frac))
+ order = list(range(len(items)))
+ random.Random(seed).shuffle(order)
+ held = set(order[:n_calib])
+ train = [it for i, it in enumerate(items) if i not in held]
+ calib = [it for i, it in enumerate(items) if i in held]
+ return train, calib
+
+
+# ----------------------------------------------------------------------------------- losses
+
+def soft_ce_loss(logits: torch.Tensor, target: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
+ """Cross-entropy against a target distribution, over each row's own options only."""
+ logp = torch.log_softmax(logits.masked_fill(~mask, _MASKED), -1)
+ return -(target * logp * mask).sum(-1).mean()
+
+
+def rlcd_loss(logits: torch.Tensor, target: torch.Tensor, mask: torch.Tensor, qtype: torch.Tensor,
+ sigma: float, samples: int = 4, w_sph: float = 0.75, w_rps: float = 1.0) -> torch.Tensor:
+ """The notebook objective: a policy-gradient term over noisy logits plus soft cross-entropy.
+
+ Zero-mean Gaussian noise (scaled by `sigma`, projected onto each row's options) is added to
+ detached logits; each sample is rewarded by `proper_reward` against the target, and the
+ normalised advantage weights the Gaussian log-density of the sample.
+ """
+ k = mask.sum(-1, keepdim=True).float()
+ fmask = mask.float()
+ eps = torch.randn((samples,) + logits.shape, device=logits.device) * sigma * fmask
+ eps = (eps - eps.sum(-1, keepdim=True) / k) * fmask
+ z = logits.detach().unsqueeze(0) + eps
+ q = torch.softmax(z.masked_fill(~mask, _MASKED), -1)
+ with torch.no_grad():
+ r = proper_reward(q, target.unsqueeze(0), qtype, fmask, w_sph=w_sph, w_rps=w_rps)
+ adv = r - r.mean(0, keepdim=True)
+ adv = adv / (adv.std() + 1e-6)
+ logp = -(((z - logits.unsqueeze(0)) ** 2) * fmask).sum(-1) / (2 * sigma ** 2)
+ return -(adv * logp).mean() + soft_ce_loss(logits, target, mask)
+
+
+def sigma_at(epoch: int, epochs: int, start: float, end: float) -> float:
+ """Exploration noise for `epoch`, annealed linearly from `start` to `end`."""
+ return start + (end - start) * (epoch / max(1, epochs - 1))
+
+
+# ------------------------------------------------------------------------------- checkpoint
+
+def resolve_device(device: Optional[str] = "auto") -> torch.device:
+ if device not in (None, "auto"):
+ return torch.device(device)
+ if torch.cuda.is_available():
+ return torch.device("cuda", 0)
+ if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available():
+ return torch.device("mps")
+ if hasattr(torch, "xpu") and torch.xpu.is_available():
+ return torch.device("xpu")
+ return torch.device("cpu")
+
+
+def load_checkpoint(model_dir: str):
+ """`(model, tokenizer, cfg)` from a local checkpoint directory, model on CPU in fp32."""
+ from safetensors.torch import load_file
+ from transformers import AutoTokenizer
+
+ from .agent import _fix_tokenizer_config
+
+ _fix_tokenizer_config(model_dir)
+ with open(os.path.join(model_dir, "rl_agent_config.json"), encoding="utf-8") as f:
+ cfg = json.load(f)
+ tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "tokenizer"))
+ model = build_model(cfg, encoder_dir=os.path.join(model_dir, "encoder"))
+ model.load_state_dict(load_file(os.path.join(model_dir, "model.safetensors")), strict=True)
+ return model.float(), tok, cfg
+
+
+def save_checkpoint(model, tok, cfg: Dict[str, Any], path: str) -> None:
+ """Write the layout `laya.load` reads: weights in fp16, encoder config, tokenizer, config."""
+ from safetensors.torch import save_file
+
+ os.makedirs(path, exist_ok=True)
+ save_file({k: v.detach().half().cpu().contiguous() for k, v in model.state_dict().items()},
+ os.path.join(path, "model.safetensors"))
+ model.encoder.config.save_pretrained(os.path.join(path, "encoder"))
+ tok.save_pretrained(os.path.join(path, "tokenizer"))
+ tmp = os.path.join(path, "rl_agent_config.json.tmp")
+ with open(tmp, "w", encoding="utf-8") as f:
+ json.dump(cfg, f, indent=2)
+ os.replace(tmp, os.path.join(path, "rl_agent_config.json"))
+
+
+# ------------------------------------------------------------------------------------ train
+
+def _forward(model, batch, device, amp: bool, detach_encoder: bool):
+ args = (batch["input_ids"].to(device), batch["attention_mask"].to(device),
+ batch["marker_pos"].to(device), batch["marker_mask"].to(device), batch["qtype"].to(device))
+ if amp:
+ with torch.autocast(device.type, dtype=torch.float16):
+ logits, _act = model(*args, detach_encoder=detach_encoder)
+ else:
+ logits, _act = model(*args, detach_encoder=detach_encoder)
+ return logits.float()
+
+
+def train_model(model, tok, items: Sequence[Dict[str, Any]], config: TrainConfig, device: torch.device,
+ max_len: int, head_max_len: int,
+ on_epoch_end: Optional[Callable[[int, float], None]] = None) -> List[float]:
+ """Train `model` in place on `items`; returns the mean loss of each epoch."""
+ config.validate()
+ if not items:
+ raise ValueError("no training items")
+ amp = (device.type == "cuda") if config.amp is None else bool(config.amp)
+ checkpointing = amp if config.gradient_checkpointing is None else bool(config.gradient_checkpointing)
+
+ if config.freeze_encoder:
+ for p in model.encoder.parameters():
+ p.requires_grad_(False)
+ elif checkpointing and hasattr(model.encoder, "gradient_checkpointing_enable"):
+ model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
+ model.head_checkpointing = checkpointing
+ model.to(device).train()
+ if config.freeze_encoder:
+ # A frozen encoder is a fixed feature extractor: dropout in it would only add noise.
+ model.encoder.eval()
+
+ groups = [{"params": [p for n, p in model.named_parameters()
+ if not n.startswith("encoder.") and p.requires_grad], "lr": config.head_lr}]
+ if not config.freeze_encoder:
+ groups.insert(0, {"params": [p for n, p in model.named_parameters()
+ if n.startswith("encoder.") and p.requires_grad], "lr": config.encoder_lr})
+ optimizer = torch.optim.AdamW(groups, weight_decay=config.weight_decay)
+ steps_per_epoch = math.ceil(len(items) / config.micro_batch)
+ updates = max(1, math.ceil(steps_per_epoch / config.grad_accum) * config.epochs)
+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=updates, eta_min=config.min_lr)
+ scaler = torch.amp.GradScaler("cuda") if amp and device.type == "cuda" else None
+
+ torch.manual_seed(config.seed)
+ order_rng = random.Random(config.seed)
+ params = [p for g in groups for p in g["params"]]
+ history = []
+ for epoch in range(config.epochs):
+ epoch_items = list(items)
+ random.Random(config.seed + epoch).shuffle(epoch_items)
+ sigma = sigma_at(epoch, config.epochs, config.sigma_start, config.sigma_end)
+ total, n_steps = 0.0, 0
+ optimizer.zero_grad(set_to_none=True)
+ for start in range(0, len(epoch_items), config.micro_batch):
+ chunk = [encode_item(tok, it, max_len, head_max_len,
+ draw_option_order(it, order_rng, config.shuffle_options))
+ for it in epoch_items[start:start + config.micro_batch]]
+ batch = collate_items([chunk], tok.pad_token_id)
+ logits = _forward(model, batch, device, amp, config.freeze_encoder)
+ mask = batch["marker_mask"].to(device)
+ target = batch["target"].to(device)
+ if config.loss == "rlcd":
+ loss = rlcd_loss(logits, target, mask, batch["qtype"].to(device), sigma,
+ config.rl_samples, config.w_sph, config.w_rps)
+ else:
+ loss = soft_ce_loss(logits, target, mask)
+ scaled = loss / config.grad_accum
+ if scaler is not None:
+ scaler.scale(scaled).backward()
+ else:
+ scaled.backward()
+ n_steps += 1
+ if n_steps % config.grad_accum == 0 or start + config.micro_batch >= len(epoch_items):
+ if scaler is not None:
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(params, config.grad_clip)
+ if scaler is not None:
+ scaler.step(optimizer)
+ scaler.update()
+ else:
+ optimizer.step()
+ scheduler.step()
+ optimizer.zero_grad(set_to_none=True)
+ total += loss.item()
+ if config.log_every and n_steps % config.log_every == 0:
+ print("epoch %d/%d step %d loss %.4f" % (epoch + 1, config.epochs, n_steps, loss.item()), flush=True)
+ mean = total / max(1, n_steps)
+ history.append(mean)
+ print("epoch %d/%d mean loss %.4f" % (epoch + 1, config.epochs, mean), flush=True)
+ if on_epoch_end is not None:
+ on_epoch_end(epoch, mean)
+ model.eval()
+ return history
+
+
+def calibration_records(model, tok, items: Sequence[Dict[str, Any]], device: torch.device,
+ max_len: int, head_max_len: int, batch_size: int = 16):
+ """`(qtype, logits, target, k)` records for `fit_temperature_map`, options in canonical order."""
+ model.eval()
+ records = []
+ for start in range(0, len(items), batch_size):
+ chunk = items[start:start + batch_size]
+ batch = collate_items([[encode_item(tok, it, max_len, head_max_len) for it in chunk]], tok.pad_token_id)
+ logits = _forward(model, batch, device, amp=False, detach_encoder=False).cpu().numpy()
+ for row, it in zip(logits, chunk):
+ records.append((it["qtype"], row[:it["k"]], it["target"], it["k"]))
+ return records
+
+
+def finetune(data: str, model_dir: str, output_dir: str, config: Optional[TrainConfig] = None,
+ device: Optional[str] = "auto") -> Dict[str, Any]:
+ """Preprocess, train, calibrate and save; returns a summary of the run.
+
+ `data` is a JSONL file of `{state, questions, gold}` rows (the schema in `docs/finetune.md`).
+ `model_dir` is a local checkpoint directory -- the layout `laya.load` reads. The result in
+ `output_dir` loads with `laya.load(output_dir)`.
+ """
+ config = config or TrainConfig()
+ config.validate()
+ dev = resolve_device(device)
+ model, tok, cfg = load_checkpoint(model_dir)
+ max_len = config.max_len or cfg.get("max_len", 512)
+ head_max_len = config.head_max_len or cfg.get("head_max_len", 192)
+
+ items, skipped = items_from_rows(tok, read_jsonl(data), max_len, head_max_len)
+ if not items:
+ raise ValueError("%s produced no training items (skipped: %r)" % (data, skipped))
+ train_items, calib_items = split_calibration(items, config.calib_max, config.calib_frac, config.calib_seed)
+ print("train items %d, calibration items %d, skipped %r, device %s"
+ % (len(train_items), len(calib_items), skipped, dev), flush=True)
+
+ def checkpoint_latest(epoch, _loss):
+ save_checkpoint(model, tok, dict(cfg, max_len=max_len, head_max_len=head_max_len),
+ os.path.join(output_dir, "checkpoint_latest"))
+
+ history = train_model(model, tok, train_items, config, dev, max_len, head_max_len,
+ on_epoch_end=checkpoint_latest)
+
+ fitted = fit_temperature_map(calibration_records(model, tok, calib_items, dev, max_len, head_max_len))
+ out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len, fine_tuned=True,
+ temperature=fitted["temperature"])
+ # An inherited bucket map takes precedence at inference and would mask the new fit.
+ out_cfg.pop("temperature_by_options", None)
+ if fitted["temperature_by_options"]:
+ out_cfg["temperature_by_options"] = fitted["temperature_by_options"]
+ out_cfg["training"] = dict(out_cfg.get("training") or {}, laya_train=asdict(config))
+ save_checkpoint(model, tok, out_cfg, output_dir)
+ return {
+ "train_items": len(train_items),
+ "calibration_items": len(calib_items),
+ "skipped": skipped,
+ "epoch_loss": history,
+ "temperature": fitted["temperature"],
+ "temperature_by_options": fitted["temperature_by_options"],
+ "n_by_bucket": fitted["n_by_bucket"],
+ "output_dir": output_dir,
+ }
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index ffe1ab20..99578629 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -84,6 +84,7 @@ SCRIPT_SUITES = [
"tests/test_langchain.py",
"tests/test_portability.py",
"tests/test_training.py",
+ "tests/test_train.py",
"tests/test_example_server_limits.py",
"tests/test_blank_lang_routing.py",
"tests/test_export_onnx_safety.py",
diff --git a/tests/test_train.py b/tests/test_train.py
new file mode 100644
index 00000000..f7ca5558
--- /dev/null
+++ b/tests/test_train.py
@@ -0,0 +1,396 @@
+"""laya.train: data prep, option-order augmentation, losses and an end-to-end run.
+
+CPU only and weight-free: the checkpoint is a one-layer BERT with a word-level tokenizer built
+in a temp directory, so nothing is downloaded.
+
+Run: python tests/test_train.py
+"""
+import json
+import os
+from pathlib import Path
+import random
+import sys
+import tempfile
+import unittest
+from unittest.mock import patch
+
+os.environ.setdefault("USE_TF", "0")
+os.environ.setdefault("USE_TORCH", "1")
+os.environ.setdefault("HF_HUB_OFFLINE", "1")
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+import numpy as np # noqa: E402
+import torch # noqa: E402
+from safetensors.torch import load_file, save_file # noqa: E402
+from tokenizers import Tokenizer # noqa: E402
+from tokenizers.models import WordLevel # noqa: E402
+from tokenizers.pre_tokenizers import Whitespace # noqa: E402
+from transformers import BertConfig, BertModel, PreTrainedTokenizerFast # noqa: E402
+
+from laya import load # noqa: E402
+from laya.common import ( # noqa: E402
+ DecisionModel,
+ QTYPES,
+ TEMP_MAX,
+ TEMP_MIN,
+ build_sequence,
+ proper_reward,
+ render_options,
+ unpermute_probs,
+)
+from laya.train import ( # noqa: E402
+ TrainConfig,
+ draw_option_order,
+ encode_item,
+ encode_state,
+ finetune,
+ items_from_rows,
+ make_item,
+ rlcd_loss,
+ soft_ce_loss,
+ split_calibration,
+ target_from_gold,
+ to_internal,
+ train_model,
+)
+
+WORDS = ["refund", "invoice", "charged", "twice", "crash", "error", "app", "screen", "please",
+ "billing", "technical", "other", "is", "it", "urgent", "low", "high", "level", "question",
+ "choice", "score", "noul", "which", "team", "yes", "no", "the", "statement", "does",
+ "not", "hold", "holds", "one", "two", "three", "four", "five", "six", "seven", ":", ",", "?"]
+
+DEPARTMENT = {"type": "choice", "instructions": "which team ?",
+ "criteria": {"billing": "", "technical": "", "other": ""}}
+URGENT = {"type": "noul", "instructions": "is it urgent ?"}
+LEVEL = {"type": "score", "instructions": "which level ?", "criteria": ["low", "high"]}
+
+BILLING_STATES = ["please refund invoice", "charged twice invoice", "refund charged twice", "invoice refund please"]
+TECH_STATES = ["app crash error", "screen error crash", "crash app screen", "error screen app"]
+
+
+def make_tokenizer():
+ vocab = {"[PAD]": 0, "[UNK]": 1, "[CLS]": 2, "[SEP]": 3, "[MASK]": 4}
+ for w in WORDS:
+ vocab[w] = len(vocab)
+ backend = Tokenizer(WordLevel(vocab, unk_token="[UNK]"))
+ backend.pre_tokenizer = Whitespace()
+ return PreTrainedTokenizerFast(tokenizer_object=backend, pad_token="[PAD]", unk_token="[UNK]",
+ cls_token="[CLS]", sep_token="[SEP]", mask_token="[MASK]")
+
+
+def make_checkpoint(path: Path, tok):
+ torch.manual_seed(0)
+ config = BertConfig(vocab_size=len(tok), hidden_size=32, num_hidden_layers=1,
+ num_attention_heads=2, intermediate_size=64, max_position_embeddings=128)
+ config.save_pretrained(path / "encoder")
+ tok.save_pretrained(path / "tokenizer")
+ model = DecisionModel(BertModel(config), head_layers=1, n_act=2)
+ save_file(model.state_dict(), path / "model.safetensors")
+ cfg = {"encoder": "unused/offline", "head_layers": 1, "act_costs": {"act": 0},
+ "max_len": 64, "head_max_len": 40, "temperature": [1.0, 1.0, 1.0],
+ "temperature_by_options": {"choice:3-5": 0.7}}
+ (path / "rl_agent_config.json").write_text(json.dumps(cfg))
+
+
+def rows(n_repeats=6):
+ out = []
+ for _ in range(n_repeats):
+ for s in BILLING_STATES:
+ out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},
+ "gold": {"department": {"probabilities": {"billing": 0.9, "technical": 0.05, "other": 0.05}},
+ "urgent": {"probabilities": {"false": 0.8, "true": 0.2}},
+ "level": {"probabilities": {"0": 0.7, "1": 0.3}}}})
+ for s in TECH_STATES:
+ out.append({"state": s, "questions": {"department": DEPARTMENT, "urgent": URGENT, "level": LEVEL},
+ "gold": {"department": {"probabilities": {"billing": 0.05, "technical": 0.9, "other": 0.05}},
+ "urgent": {"probabilities": {"false": 0.2, "true": 0.8}},
+ "level": {"probabilities": {"0": 0.2, "1": 0.8}}}})
+ return out
+
+
+class DataTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tok = make_tokenizer()
+
+ def item(self, question, target, state="app crash", head_max_len=40):
+ item, reason = make_item(self.tok, to_internal("q", question), target,
+ encode_state(self.tok, state, 64), head_max_len)
+ self.assertIsNone(reason)
+ return item
+
+ def test_target_from_gold_orders_and_normalises(self):
+ def target(question, probs):
+ return target_from_gold(to_internal("q", question), {"probabilities": probs})
+
+ self.assertEqual(target(DEPARTMENT, {"technical": 2.0, "billing": 2.0}), [0.5, 0.5, 0.0])
+ self.assertEqual(target(URGENT, {"true": 0.75, "false": 0.25}), [0.25, 0.75])
+ self.assertEqual(target(LEVEL, {"1": 1.0}), [0.0, 1.0])
+ self.assertEqual(target(DEPARTMENT, {}), [1 / 3] * 3)
+ # list-form choice and non-string labels, which inference accepts
+ self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},
+ {"screen": 1.0}), [0.0, 1.0])
+ self.assertEqual(target({"type": "choice", "instructions": "which ?", "criteria": {1: "", 2: ""}},
+ {"2": 3.0, "1": 1.0}), [0.25, 0.75])
+ for bad in ({"billing": -1.0}, {"billing": float("nan")}, {"billing": "high"}):
+ with self.assertRaises(ValueError):
+ target(DEPARTMENT, bad)
+
+ def test_questions_are_rendered_exactly_as_inference_renders_them(self):
+ # The sequence a training item produces must be the one `Agent` builds for the same
+ # question, or the model is trained on option texts it never sees.
+ from laya.agent import Agent
+
+ questions = [
+ DEPARTMENT, URGENT, LEVEL,
+ {"type": "choice", "instructions": "which ?", "criteria": ["app", "screen"]},
+ {"type": "noul", "instructions": "is it ?", "labels": {"false": "no", "true": "yes"},
+ "criteria": {True: "holds", False: "does not hold"}},
+ {"type": "noul", "instructions": {"which": "team"}},
+ ]
+ for question in questions:
+ with self.subTest(question=question):
+ k = len(render_options(Agent._to_internal(question)))
+ item = self.item(question, [1.0 / k] * k)
+ expected = build_sequence(self.tok, "app crash", Agent._to_internal(question), 64, 40)
+ enc = encode_item(self.tok, item, 64, 40)
+ self.assertEqual((enc["ids"], enc["markers"]), expected)
+
+ def test_rows_share_one_tokenized_state(self):
+ items, skipped = items_from_rows(self.tok, rows(1)[:1], 64, 40)
+ self.assertEqual((len(items), skipped), (3, {}))
+ self.assertTrue(items[0]["state_ids"] is items[1]["state_ids"] is items[2]["state_ids"])
+ self.assertEqual([it["qtype"] for it in items], [QTYPES["choice"], QTYPES["noul"], QTYPES["score"]])
+
+ def test_question_without_gold_is_not_an_item_or_a_skip(self):
+ row = dict(rows(1)[0])
+ row["gold"] = {"department": row["gold"]["department"]}
+ items, skipped = items_from_rows(self.tok, [row], 64, 40)
+ self.assertEqual((len(items), skipped), (1, {}))
+
+ def test_bad_rows_are_skipped_with_a_reason(self):
+ row = dict(rows(1)[0])
+ row["questions"] = dict(row["questions"],
+ broken={"type": "choice", "instructions": "which ?", "criteria": []},
+ negative=DEPARTMENT)
+ row["gold"] = dict(row["gold"], broken={"probabilities": {}},
+ negative={"probabilities": {"billing": -2.0}})
+ items, skipped = items_from_rows(self.tok, [row], 64, 40)
+ self.assertEqual((len(items), skipped), (3, {"invalid_question": 1, "invalid_target": 1}))
+
+ def test_options_the_head_budget_collapses_are_skipped(self):
+ # Two options that only differ after the per-option cap come out as the same span (#538).
+ q = to_internal("q", {"type": "choice", "instructions": "which ?",
+ "criteria": {"one two three four five six": "", "one two three four five seven": ""}})
+ state_ids = encode_state(self.tok, "app", 64)
+ self.assertEqual(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=20),
+ (None, "options_collapsed"))
+ self.assertIsNotNone(make_item(self.tok, q, [0.5, 0.5], state_ids, head_max_len=64)[0])
+
+ def test_target_length_must_match_options(self):
+ state_ids = encode_state(self.tok, "app", 64)
+ self.assertEqual(make_item(self.tok, to_internal("q", DEPARTMENT), [0.5, 0.5], state_ids, 40),
+ (None, "target_mismatch"))
+
+ def test_canonical_encoding_matches_build_sequence(self):
+ item = self.item(DEPARTMENT, [0.8, 0.1, 0.1])
+ enc = encode_item(self.tok, item, 64, 40)
+ ids, markers = build_sequence(self.tok, "app crash", item["q"], 64, 40)
+ self.assertEqual((enc["ids"], enc["markers"], enc["target"]), (ids, markers, [0.8, 0.1, 0.1]))
+
+ def test_permuted_item_carries_a_permuted_target(self):
+ item = self.item(DEPARTMENT, [0.7, 0.2, 0.1])
+ order = [2, 0, 1]
+ enc = encode_item(self.tok, item, 64, 40, option_order=order)
+ self.assertEqual(enc["target"], [0.1, 0.7, 0.2])
+ labels = list(DEPARTMENT["criteria"])
+ # The token right after slot s's [MASK] is the label of option order[s].
+ for slot, marker in enumerate(enc["markers"]):
+ self.assertEqual(enc["ids"][marker], self.tok.mask_token_id)
+ self.assertEqual(enc["ids"][marker + 1], self.tok.convert_tokens_to_ids(labels[order[slot]]))
+
+ def test_permuted_training_input_is_what_inference_scores_for_that_order(self):
+ # Inference with option_order builds the same tokens in the same slots, and
+ # unpermute_probs maps the slot-ordered target back to the canonical one.
+ from laya.agent import Agent
+
+ order = [1, 2, 0]
+ item = self.item(DEPARTMENT, [0.6, 0.3, 0.1])
+ enc = encode_item(self.tok, item, 64, 40, option_order=order)
+ internal = Agent._to_internal(dict(DEPARTMENT, option_order=order))
+ expected_ids, _ = build_sequence(self.tok, "app crash", internal, 64, 40,
+ option_order=internal["option_order"])
+ self.assertEqual(enc["ids"], expected_ids)
+ self.assertEqual(list(unpermute_probs(np.array(enc["target"]), order)), item["target"])
+
+ def test_only_configured_types_are_shuffled(self):
+ state_ids = encode_state(self.tok, "app", 64)
+ choice, _ = make_item(self.tok, to_internal("q", DEPARTMENT), [1.0, 0.0, 0.0], state_ids, 40)
+ noul, _ = make_item(self.tok, to_internal("q", URGENT), [0.5, 0.5], state_ids, 40)
+ self.assertIsNone(draw_option_order(noul, random.Random(0), ("choice",)))
+ orders = {tuple(draw_option_order(choice, random.Random(s), ("choice",))) for s in range(20)}
+ self.assertTrue(all(sorted(o) == [0, 1, 2] for o in orders))
+ self.assertGreater(len(orders), 1)
+ self.assertEqual(draw_option_order(choice, random.Random(3), ("choice",)),
+ draw_option_order(choice, random.Random(3), ("choice",)))
+ self.assertIsNone(draw_option_order(choice, random.Random(0), ()))
+
+ def test_calibration_split_is_disjoint_and_seeded(self):
+ items = list(range(50))
+ train, calib = split_calibration(items, calib_max=400, calib_frac=0.1, seed=7)
+ self.assertEqual((len(train), len(calib)), (45, 5))
+ self.assertEqual(sorted(train + calib), items)
+ self.assertEqual(split_calibration(items, 400, 0.1, 7), (train, calib))
+ self.assertEqual(len(split_calibration(items, 3, 0.5, 7)[1]), 3)
+
+
+class LossTests(unittest.TestCase):
+ def test_soft_ce_ignores_padded_options(self):
+ logits = torch.tensor([[2.0, 0.5, 9.0], [1.0, -1.0, 0.0]])
+ target = torch.tensor([[0.75, 0.25, 0.0], [0.2, 0.3, 0.5]])
+ mask = torch.tensor([[True, True, False], [True, True, True]])
+ manual = -(torch.tensor([0.75, 0.25]) * torch.log_softmax(torch.tensor([2.0, 0.5]), -1)).sum()
+ manual = (manual - (target[1] * torch.log_softmax(logits[1], -1)).sum()) / 2
+ self.assertAlmostEqual(soft_ce_loss(logits, target, mask).item(), manual.item(), places=5)
+
+ def test_rlcd_is_finite_and_differentiable(self):
+ torch.manual_seed(0)
+ logits = torch.randn(4, 3, requires_grad=True)
+ target = torch.softmax(torch.randn(4, 3), -1)
+ mask = torch.ones(4, 3, dtype=torch.bool)
+ loss = rlcd_loss(logits, target, mask, torch.tensor([0, 1, 2, 0]), sigma=0.4)
+ loss.backward()
+ self.assertTrue(torch.isfinite(loss))
+ self.assertTrue(torch.isfinite(logits.grad).all())
+
+ def test_rlcd_matches_the_notebook_objective(self):
+ # The loss as written in research/scripts/finetune_single_device.py, inlined: with the
+ # same RNG state, rlcd_loss must give the same value and the same gradient.
+ def script_loss(logits, target, mask, qtype, sigma):
+ k = mask.sum(-1, keepdim=True).float()
+ eps = torch.randn((4,) + logits.shape) * sigma * mask
+ eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
+ z = logits.detach().unsqueeze(0) + eps
+ q = torch.softmax(z.masked_fill(~mask, -1e4), -1)
+ with torch.no_grad():
+ r = proper_reward(q, target.unsqueeze(0), qtype, mask, w_sph=0.75, w_rps=1.0)
+ adv = r - r.mean(0, keepdim=True)
+ adv = adv / (adv.std() + 1e-6)
+ logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)
+ ce = (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()
+ return -(adv * logp).mean() - ce
+
+ torch.manual_seed(1)
+ base = torch.randn(5, 4)
+ mask = torch.tensor([[1, 1, 1, 1], [1, 1, 0, 0], [1, 1, 1, 0], [1, 1, 1, 1], [1, 1, 0, 0]],
+ dtype=torch.bool)
+ target = torch.softmax(torch.randn(5, 4), -1) * mask
+ target = target / target.sum(-1, keepdim=True)
+ qtype = torch.tensor([0, 2, 1, 0, 2])
+ results = []
+ for fn in (lambda lg: script_loss(lg, target, mask, qtype, 0.3),
+ lambda lg: rlcd_loss(lg, target, mask, qtype, 0.3)):
+ logits = base.masked_fill(~mask, -1e4).clone().requires_grad_(True)
+ torch.manual_seed(5)
+ loss = fn(logits)
+ loss.backward()
+ results.append((loss.item(), logits.grad.clone()))
+ self.assertAlmostEqual(results[0][0], results[1][0], places=5)
+ self.assertTrue(torch.allclose(results[0][1], results[1][1], atol=1e-6))
+
+ def test_defaults_train_like_the_notebook(self):
+ config = TrainConfig()
+ self.assertEqual((config.loss, config.shuffle_options), ("rlcd", ()))
+ self.assertEqual(config.micro_batch * config.grad_accum, 64)
+
+ def test_config_rejects_unknown_values(self):
+ for bad in (TrainConfig(loss="mse"), TrainConfig(shuffle_options=("choise",)),
+ TrainConfig(epochs=0), TrainConfig(calib_frac=1.0)):
+ with self.assertRaises(ValueError):
+ bad.validate()
+
+
+class EndToEndTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.tmp = tempfile.TemporaryDirectory()
+ cls.addClassCleanup(cls.tmp.cleanup)
+ cls.root = Path(cls.tmp.name)
+ cls.tok = make_tokenizer()
+ make_checkpoint(cls.root / "base", cls.tok)
+ cls.data = cls.root / "train.jsonl"
+ cls.data.write_text("\n".join(json.dumps(r) for r in rows()), encoding="utf-8")
+
+ def run_finetune(self, out, **overrides):
+ # Options stay in order here: a randomly initialised one-layer encoder cannot learn to read
+ # option labels in a handful of epochs, only their slots. The permutation itself is
+ # covered by DataTests; test_shuffled_run_saves_a_loadable_checkpoint runs it end to end.
+ kwargs = dict(epochs=12, micro_batch=8, grad_accum=1, encoder_lr=1e-3, head_lr=1e-3,
+ calib_frac=0.2, log_every=0, loss="soft-ce", shuffle_options=())
+ kwargs.update(overrides)
+ config = TrainConfig(**kwargs)
+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
+ return finetune(str(self.data), str(self.root / "base"), str(self.root / out), config, device="cpu")
+
+ def test_finetune_learns_and_saves_a_loadable_checkpoint(self):
+ report = self.run_finetune("out")
+ self.assertEqual(report["train_items"] + report["calibration_items"], len(rows()) * 3)
+ self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])
+
+ saved = json.loads((self.root / "out" / "rl_agent_config.json").read_text())
+ self.assertTrue(saved["fine_tuned"])
+ self.assertEqual((saved["max_len"], saved["head_max_len"]), (64, 40))
+ # The inherited bucket would override the new fit at inference; too few items to refit it.
+ self.assertNotIn("temperature_by_options", saved)
+ self.assertTrue(all(TEMP_MIN <= t <= TEMP_MAX for t in saved["temperature"]))
+ self.assertEqual(saved["training"]["laya_train"]["loss"], "soft-ce")
+ self.assertTrue((self.root / "out" / "checkpoint_latest" / "model.safetensors").exists())
+
+ with patch("huggingface_hub.snapshot_download", side_effect=AssertionError("unexpected download")):
+ agent = load(str(self.root / "out"), device="cpu")
+ answers = agent.predict("please refund invoice", {"department": DEPARTMENT})["answers"]
+ self.assertEqual(answers["department"]["choice"], "billing")
+ answers = agent.predict("app crash error", {"department": DEPARTMENT})["answers"]
+ self.assertEqual(answers["department"]["choice"], "technical")
+
+ def test_shuffled_run_saves_a_loadable_checkpoint(self):
+ report = self.run_finetune("out_shuffled", shuffle_options=("choice",), epochs=1)
+ saved = json.loads((self.root / "out_shuffled" / "rl_agent_config.json").read_text())
+ self.assertEqual(saved["training"]["laya_train"]["shuffle_options"], ["choice"])
+ self.assertEqual(len(report["epoch_loss"]), 1)
+ agent = load(str(self.root / "out_shuffled"), device="cpu")
+ self.assertIn(agent.predict("app", {"department": DEPARTMENT})["answers"]["department"]["choice"],
+ DEPARTMENT["criteria"])
+
+ def test_rlcd_objective_still_trains(self):
+ report = self.run_finetune("out_rlcd", loss="rlcd")
+ self.assertLess(report["epoch_loss"][-1], report["epoch_loss"][0])
+
+ def test_frozen_encoder_is_left_untouched(self):
+ self.run_finetune("out_frozen", freeze_encoder=True, epochs=2)
+ before = load_file(str(self.root / "base" / "model.safetensors"))
+ after = load_file(str(self.root / "out_frozen" / "model.safetensors"))
+ for name in before:
+ if name.startswith("encoder."):
+ self.assertTrue(torch.allclose(before[name].half().float(), after[name].float()), name)
+ self.assertFalse(torch.allclose(before["scorer.1.weight"].half().float(), after["scorer.1.weight"].float()))
+
+ def test_frozen_encoder_runs_without_dropout(self):
+ from laya.train import load_checkpoint
+
+ model, tok, _cfg = load_checkpoint(str(self.root / "base"))
+ modes = []
+ model.encoder.register_forward_pre_hook(lambda module, _args: modes.append(module.training))
+ items, _ = items_from_rows(tok, rows(1)[:4], 64, 40)
+ train_model(model, tok, items, TrainConfig(epochs=1, freeze_encoder=True, log_every=0),
+ torch.device("cpu"), 64, 40)
+ self.assertTrue(modes)
+ self.assertFalse(any(modes))
+
+ def test_train_model_rejects_empty_input(self):
+ with self.assertRaises(ValueError):
+ train_model(None, self.tok, [], TrainConfig(), torch.device("cpu"), 64, 40)
+
+
+if __name__ == "__main__":
+ unittest.main()