Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
refactor(finetune): converge single-process wrappers on laya.train
Bruce-Yiicebf8b16a5bc
diff --git a/notebooks/laya_finetune_typed_decisions_mps.py b/notebooks/laya_finetune_typed_decisions_mps.pyindex 6f854be3..17ab53f1 100644--- a/notebooks/laya_finetune_typed_decisions_mps.py+++ b/notebooks/laya_finetune_typed_decisions_mps.py@@ -16,17 +16,12 @@ Useful examples:import argparseimport gcimport json-import math-import randomfrom pathlib import Pathimport torchfrom huggingface_hub import snapshot_download-from safetensors.torch import load_file, save_file-from transformers import AutoTokenizerfrom laya.agent import _fix_tokenizer_config-from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, build_model, build_sequence, proper_reward, render_optionsMODEL_ID = "convaiinnovations/laya"DATASET_ID = "LocalLLaMA/typed-decisions"@@ -55,316 +50,97 @@ def prepare_model(model_dir):return str(model_dir)-def build_training_item(tokenizer, cfg, state, question, gold_question):- qtype = question["type"]- criteria = question.get("criteria", {})-- if qtype == "choice":- keys = list(criteria.keys())- target = [gold_question["probabilities"].get(k, 0.0) for k in keys]- elif qtype == "noul":- target = [- gold_question["probabilities"].get("false", 0.5),- gold_question["probabilities"].get("true", 0.5),- ]- elif qtype == "score":- n_levels = len(criteria) if isinstance(criteria, list) else 4- target = [gold_question["probabilities"].get(str(i), 0.0) for i in range(n_levels)]- else:- return None-- total = sum(target)- if total > 0:- target = [float(x) / total for x in target]- else:- target = [1.0 / len(target)] * len(target)-- label = target.index(max(target))- n_options = len(render_options({"t": qtype, "crit": criteria}))- sequence, markers = build_sequence(- tokenizer,- state,- {"t": qtype, "ins": question["instructions"], "crit": criteria},- cfg["max_len"],- cfg["head_max_len"],- )- if len(markers) != n_options:- return None-- return {- "ids": sequence,- "markers": markers,- "qtype": QTYPES[qtype],- "target": target,- "label": label,- }--def prepare_items(model_dir, items_path, force=False):+ # The convergence target is `laya.train`; its input is row-level JSONL, not+ # pre-tokenized item tensors. Keep the cache honest: when the requested cache+ # changes, also invalidate the row cache.items_path = Path(items_path)+ rows_path = items_path.with_suffix(items_path.suffix + '.rows.jsonl')model_path = Path(model_dir).resolve()- with open(model_path / "rl_agent_config.json") as f:+ with open(model_path / 'rl_agent_config.json') as f:cfg = json.load(f)- max_len = cfg.get("max_len", 1024)- head_max_len = cfg.get("head_max_len", 256)- cache_meta_path = items_path.with_name(items_path.name + ".meta.json")+ max_len = cfg.get('max_len', 1024)+ head_max_len = cfg.get('head_max_len', 256)+ cache_meta_path = rows_path.with_name(rows_path.name + '.meta.json')cache_key = {- "model_dir": str(model_path),- "max_len": max_len,- "head_max_len": head_max_len,- "dataset": DATASET_ID,- "split": "train",+ 'model_dir': str(model_path),+ 'max_len': max_len,+ 'head_max_len': head_max_len,+ 'dataset': DATASET_ID,+ 'split': 'train',+ 'format': 'rows_jsonl',}- if items_path.exists() and not force:+ if rows_path.exists() and not force:if not cache_meta_path.exists():- # Legacy item files predate the sidecar metadata. They remain- # usable offline; newly written caches always receive a key.- print(f"Using legacy cached training items without metadata: {items_path}")- return+ print(f'Using legacy cached training rows without metadata: {rows_path}')+ return str(rows_path)try:with open(cache_meta_path) as f:cached_key = json.load(f)except (OSError, json.JSONDecodeError):cached_key = Noneif cached_key == cache_key:- print(f"Using cached training items: {items_path}")- return- print("Training-item cache key changed; rebuilding the cache.")+ print(f'Using cached training rows: {rows_path}')+ return str(rows_path)+ print('Training-row cache key changed; rebuilding the cache.')- # Keep datasets optional when a compatible local cache is already available.from datasets import load_dataset- tokenizer = AutoTokenizer.from_pretrained(model_path / "tokenizer")- print(f"Downloading dataset {DATASET_ID} ...")- dataset = load_dataset(DATASET_ID, "all", split="train")-- items = []- skipped = 0- for row in dataset:- state = json.loads(row["state"])- questions = json.loads(row["questions"])- gold = json.loads(row["gold"])- for qid, question in questions.items():- if qid not in gold:- continue- item = build_training_item(- tokenizer,- {**cfg, "max_len": max_len, "head_max_len": head_max_len},- state,- question,- gold[qid],- )- if item is None:- skipped += 1- else:- items.append(item)-- items_path.parent.mkdir(parents=True, exist_ok=True)- torch.save(items, items_path)- with open(cache_meta_path, "w") as f:+ print(f'Downloading dataset {DATASET_ID} ...')+ dataset = load_dataset(DATASET_ID, 'all', split='train')+ rows_path.parent.mkdir(parents=True, exist_ok=True)+ n_rows = 0+ with open(rows_path, 'w', encoding='utf-8') as f:+ for row in dataset:+ state = json.loads(row['state'])+ questions = json.loads(row['questions'])+ gold = json.loads(row['gold'])+ f.write(json.dumps({'state': state, 'questions': questions, 'gold': gold}, ensure_ascii=False) + '\n')+ n_rows += 1+ with open(cache_meta_path, 'w') as f:json.dump(cache_key, f, indent=2)- print(f"Saved {len(items)} training items to {items_path}; skipped={skipped}")---def collate(items, pad_id):- batch_size = len(items)- seq_len = max(len(item["ids"]) for item in items)- kmax = max(len(item["markers"]) for item in items)- input_ids = torch.full((batch_size, seq_len), pad_id, dtype=torch.long)- attention = torch.zeros((batch_size, seq_len), dtype=torch.long)- marker_pos = torch.zeros((batch_size, kmax), dtype=torch.long)- marker_mask = torch.zeros((batch_size, kmax), dtype=torch.bool)- target = torch.zeros((batch_size, kmax), dtype=torch.float32)-- for i, item in enumerate(items):- length = len(item["ids"])- input_ids[i, :length] = torch.tensor(item["ids"], dtype=torch.long)- attention[i, :length] = 1- k = len(item["markers"])- marker_pos[i, :k] = torch.tensor(item["markers"], dtype=torch.long)- marker_mask[i, :k] = True- target[i, :len(item["target"])] = torch.tensor(item["target"], dtype=torch.float32)-- return (- input_ids,- attention,- marker_pos,- marker_mask,- target,- torch.tensor([item["qtype"] for item in items], dtype=torch.long),- )---def fit_temperature(samples):- if len(samples) < 10:- return 1.0- kmax = max(len(logits) for logits, _ in samples)- logits = torch.full((len(samples), kmax), -1e4)- targets = torch.zeros((len(samples), kmax))- for i, (values, target) in enumerate(samples):- logits[i, :len(values)] = torch.as_tensor(values)- targets[i, :len(target)] = torch.as_tensor(target, dtype=torch.float32)-- log_temperature = torch.zeros(1, requires_grad=True)- optimizer = torch.optim.LBFGS([log_temperature], lr=0.1, max_iter=100)-- def closure():- optimizer.zero_grad()- loss = -(targets * torch.log_softmax(logits / log_temperature.exp(), -1)).sum(-1).mean()- loss.backward()- return loss-- optimizer.step(closure)- return float(torch.clamp(log_temperature.exp(), TEMP_MIN, TEMP_MAX).item())---def save_checkpoint(model, tokenizer, cfg, output_dir, epoch, final=False):- path = Path(output_dir) if final else Path(output_dir) / "checkpoint_latest"- path.mkdir(parents=True, exist_ok=True)- weights = {- name: value.detach().half().cpu().contiguous()- for name, value in model.state_dict().items()- }- save_file(weights, str(path / "model.safetensors"))- model.encoder.config.save_pretrained(path / "encoder")- tokenizer.save_pretrained(path / "tokenizer")- with open(path / "checkpoint_meta.json", "w") as f:- json.dump({"epoch": epoch, "final": final}, f, indent=2)- with open(path / "rl_agent_config.json", "w") as f:- json.dump(cfg, f, indent=2)---def train(args, model_dir, items_path, device):- with open(Path(model_dir) / "rl_agent_config.json") as f:- cfg = json.load(f)- cfg.update({"max_tokens_per_batch": 2048, "max_len": 1024, "head_max_len": 256})- if not args.no_checkpointing:- cfg["gradient_checkpointing"] = True-- tokenizer = AutoTokenizer.from_pretrained(Path(model_dir) / "tokenizer")- model = build_model(cfg, encoder_dir=Path(model_dir) / "encoder")- model.load_state_dict(load_file(str(Path(model_dir) / "model.safetensors")), strict=True)- model.float()-- if not args.no_checkpointing:- model.encoder.gradient_checkpointing_enable(- gradient_checkpointing_kwargs={"use_reentrant": False}- )- model.head_checkpointing = True- model.to(device).train()-- all_items = torch.load(items_path, map_location="cpu", weights_only=False)- order = list(range(len(all_items)))- random.Random(20260922).shuffle(order)- n_calib = min(args.calib_max, len(all_items) // 10)- calib_items = [all_items[i] for i in sorted(order[:n_calib])]- train_items = [all_items[i] for i in sorted(order[n_calib:])]-- encoder_params = [p for n, p in model.named_parameters() if "encoder." in n]- head_params = [p for n, p in model.named_parameters() if "encoder." not in n]- optimizer = torch.optim.AdamW(- [{"params": encoder_params, "lr": 2.5e-5}, {"params": head_params, "lr": 1e-4}],- weight_decay=0.01,+ print(f'Saved {n_rows} training rows to {rows_path}')+ return str(rows_path)+++def train(args, model_dir, rows_path, device):+ from laya.train import TrainConfig, finetune++ config = TrainConfig(+ epochs=args.epochs,+ micro_batch=args.micro_batch,+ grad_accum=args.grad_accum,+ calib_max=args.calib_max,+ calib_seed=20260922,+ seed=42,+ max_len=1024,+ head_max_len=256,+ gradient_checkpointing=not args.no_checkpointing,+ amp=False,)- updates = max(1, math.ceil(len(train_items) / args.micro_batch / args.grad_accum) * args.epochs)- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(- optimizer, T_max=updates, eta_min=1e-6+ summary = finetune(+ data=str(rows_path),+ model_dir=str(model_dir),+ output_dir=str(args.output_dir),+ config=config,+ device=args.device if args.device != 'auto' else ('mps' if torch.backends.mps.is_available() else 'cpu'),)-- print(f"Device: {device}")- print(f"Training items: {len(train_items)}; calibration items: {len(calib_items)}")- print(f"micro_batch={args.micro_batch}; grad_accum={args.grad_accum}; epochs={args.epochs}")-- for epoch in range(args.epochs):- random.Random(42 + epoch).shuffle(train_items)- optimizer.zero_grad(set_to_none=True)- total_loss = 0.0- n_batches = 0- sigma = 0.4 + (0.1 - 0.4) * epoch / max(1, args.epochs - 1)-- for start in range(0, len(train_items), args.micro_batch):- chunk = train_items[start:start + args.micro_batch]- ids, attention, positions, mask, target, qtype = collate(chunk, tokenizer.pad_token_id)- ids, attention = ids.to(device), attention.to(device)- positions = positions.to(device)- mask = mask.to(device)- target = target.to(device)- qtype = qtype.to(device)-- logits, activation = model(ids, attention, positions, mask, qtype)- logits = logits.float()- k = mask.sum(-1, keepdim=True).float()-- eps = torch.randn((4,) + logits.shape, device=device) * sigma * mask- eps = (eps - eps.sum(-1, keepdim=True) / k) * mask- noisy_logits = logits.detach().unsqueeze(0) + eps- probabilities = torch.softmax(noisy_logits.masked_fill(~mask, -1e4), -1)-- with torch.no_grad():- reward = proper_reward(- probabilities,- target.unsqueeze(0),- qtype,- mask,- w_sph=0.75,- w_rps=1.0,- )- advantage = reward - reward.mean(0, keepdim=True)- advantage = advantage / (advantage.std() + 1e-6)-- logp = -(((noisy_logits - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma**2)- loss_rl = -(advantage * logp).mean()- loss_ce = -(- target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)- ).sum(-1).mean()- loss = (loss_rl + loss_ce + 0.0 * activation.sum()) / args.grad_accum- loss.backward()-- n_batches += 1- if n_batches % args.grad_accum == 0 or start + args.micro_batch >= len(train_items):- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)- optimizer.step()- scheduler.step()- optimizer.zero_grad(set_to_none=True)-- total_loss += loss.item() * args.grad_accum- if n_batches % 100 == 0:- print(f"epoch {epoch + 1}/{args.epochs}, step {n_batches}, loss={loss.item() * args.grad_accum:.4f}")-- avg_loss = total_loss / max(1, n_batches)- print(f"Epoch {epoch + 1}/{args.epochs} complete; avg_loss={avg_loss:.4f}")- save_checkpoint(model, tokenizer, cfg, args.output_dir, epoch + 1)-- print("Running temperature calibration ...")- model.eval()- samples = [[] for _ in range(3)]- with torch.no_grad():- for start in range(0, len(calib_items), args.micro_batch):- chunk = calib_items[start:start + args.micro_batch]- ids, attention, positions, mask, target, qtype = collate(chunk, tokenizer.pad_token_id)- logits, _ = model(- ids.to(device), attention.to(device), positions.to(device), mask.to(device), qtype.to(device)- )- for i, item in enumerate(chunk):- qtype_id = item["qtype"]- samples[qtype_id].append(- (logits[i, :len(item["markers"])].cpu(), item["target"])- )-- temperatures = [fit_temperature(group) if group else 1.2 for group in samples]- save_checkpoint(model, tokenizer, cfg, args.output_dir, args.epochs, final=True)- cfg.update({- "fine_tuned": True,- "model_name": "laya-typed-decisions",- "temperature": temperatures,- })- cfg.pop("temperature_by_options", None)- with open(Path(args.output_dir) / "rl_agent_config.json", "w") as f:- json.dump(cfg, f, indent=2)- with open(Path(args.output_dir) / "checkpoint_latest" / "rl_agent_config.json", "w") as f:- json.dump(cfg, f, indent=2)- print(f"Model saved to {args.output_dir}")- print(f"Temperatures: {temperatures}")+ # Wrapper-visible metadata and persistence preserved from the old entry point.+ for rel in [Path(args.output_dir) / 'rl_agent_config.json', Path(args.output_dir) / 'checkpoint_latest' / 'rl_agent_config.json']:+ if rel.exists():+ cfg = json.loads(rel.read_text(encoding='utf-8'))+ for name, entry in (summary.get('calibration') or {}).items():+ if isinstance(entry, dict) and entry.get('items') == 0:+ idx = {'choice': 0, 'score': 1, 'noul': 2}.get(name)+ if idx is not None:+ cfg['temperature'][idx] = 1.2+ cfg.pop('temperature_by_options', None)+ cfg['model_name'] = 'laya-typed-decisions'+ cfg['max_tokens_per_batch'] = 2048+ rel.write_text(json.dumps(cfg, indent=2), encoding='utf-8')+ print(f'Train items: {summary["train_items"]}; calibration items: {summary["calibration_items"]}')+ print('Fitted temperatures (choice, score, noul):', [round(t, 3) for t in summary['temperature']])+ print(f'Model saved to {args.output_dir}')def main():@@ -395,8 +171,8 @@ def main():torch.set_float32_matmul_precision("high")device = choose_device(args.device)model_dir = prepare_model(args.model_dir)- prepare_items(model_dir, args.items, force=args.force_preprocess)- train(args, model_dir, args.items, device)+ rows_path = prepare_items(model_dir, args.items, force=args.force_preprocess)+ train(args, model_dir, rows_path, device)gc.collect()diff --git a/research/scripts/finetune_single_device.py b/research/scripts/finetune_single_device.pyindex fb5af748..ce842618 100644--- a/research/scripts/finetune_single_device.py+++ b/research/scripts/finetune_single_device.py@@ -17,143 +17,16 @@ Usage:import argparseimport jsonimport os-import randomimport sys+from pathlib import PathROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))sys.path.insert(0, ROOT)-import torchfrom huggingface_hub import snapshot_download-from safetensors.torch import load_file, save_file-from transformers import AutoTokenizerfrom laya.agent import _fix_tokenizer_config-from laya.common import (- QTYPES,- TEMP_MAX,- TEMP_MIN,- build_model,- build_sequence,- proper_reward,- render_options,-)---def build_training_item(tok, cfg, state, q, gold_q):- t = q["type"]- crit = q.get("criteria", {})- if t == "choice":- keys = list(crit.keys())- target = [gold_q["probabilities"].get(k, 0.0) for k in keys]- elif t == "noul":- target = [- gold_q["probabilities"].get("false", 0.5),- gold_q["probabilities"].get("true", 0.5),- ]- else:- n_levels = len(crit) if isinstance(crit, list) else 4- target = [gold_q["probabilities"].get(str(i), 0.0) for i in range(n_levels)]- total = sum(target)- target = (- [v / total for v in target] if total > 0 else [1.0 / len(target)] * len(target)- )- seq, markers = build_sequence(- tok,- state,- {"t": t, "ins": q["instructions"], "crit": crit},- cfg["max_len"],- cfg["head_max_len"],- )- if len(markers) != len(render_options({"t": t, "crit": crit})):- return None- return {- "ids": seq,- "markers": markers,- "qtype": QTYPES[t],- "target": target,- "label": target.index(max(target)),- }---def preprocess(tok, cfg, data_path):- items = []- with open(data_path, encoding="utf-8") as f:- for line in f:- row = json.loads(line)- for qid, q in row["questions"].items():- if qid not in row["gold"]:- continue- item = build_training_item(tok, cfg, row["state"], q, row["gold"][qid])- if item is not None:- items.append(item)- return items---def collate(items, pad_id):- n, length = len(items), max(len(it["ids"]) for it in items)- kmax = max(len(it["markers"]) for it in items)- ids = torch.full((n, length), pad_id, dtype=torch.long)- att = torch.zeros((n, length), dtype=torch.long)- mpos = torch.zeros((n, kmax), dtype=torch.long)- mmask = torch.zeros((n, kmax), dtype=torch.bool)- target = torch.zeros((n, kmax), dtype=torch.float32)- for i, it in enumerate(items):- ids[i, : len(it["ids"])] = torch.tensor(it["ids"])- att[i, : len(it["ids"])] = 1- k = len(it["markers"])- mpos[i, :k] = torch.tensor(it["markers"])- mmask[i, :k] = True- target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)- return {- "input_ids": ids,- "attention_mask": att,- "marker_pos": mpos,- "marker_mask": mmask,- "target": target,- "qtype": torch.tensor([it["qtype"] for it in items]),- }---def forward(model, batch, device, use_amp):- if use_amp:- with torch.autocast("cuda", dtype=torch.float16):- return model(- batch["input_ids"].to(device),- batch["attention_mask"].to(device),- batch["marker_pos"].to(device),- batch["marker_mask"].to(device),- batch["qtype"].to(device),- )- return model(- batch["input_ids"].to(device),- batch["attention_mask"].to(device),- batch["marker_pos"].to(device),- batch["marker_mask"].to(device),- batch["qtype"].to(device),- )---def fit_one_temp(sel):- if len(sel) < 10:- return 1.0- kmax = max(len(z) for z, _ in sel)- Z = torch.full((len(sel), kmax), -1e4)- T = torch.zeros((len(sel), kmax))- for i, (z, t) in enumerate(sel):- Z[i, : len(z)] = torch.tensor(z)- T[i, : len(t)] = torch.tensor(t, dtype=torch.float32)- log_t = torch.zeros(1, requires_grad=True)- opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)-- def closure():- opt.zero_grad()- loss = -(T * torch.log_softmax(Z / log_t.exp(), -1)).sum(-1).mean()- loss.backward()- return loss-- opt.step(closure)- return float(torch.clamp(log_t.exp(), TEMP_MIN, TEMP_MAX).item())+from laya.train import TrainConfig, finetunedef main():@@ -176,20 +49,6 @@ def main():parser.add_argument("--seed", type=int, default=0)args = parser.parse_args()- if args.device == "auto":- device = (- torch.device("cuda", 0)- if torch.cuda.is_available()- else torch.device("cpu")- )- else:- device = torch.device(args.device)- use_amp = device.type == "cuda"- torch.manual_seed(args.seed)- random.seed(args.seed)- torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", os.cpu_count() or 1)))- print("Device:", device)-if args.model_dir:model_dir = args.model_direlse:@@ -202,153 +61,41 @@ def main():)_fix_tokenizer_config(model_dir)- with open(os.path.join(model_dir, "rl_agent_config.json")) as f:- cfg = json.load(f)- cfg["gradient_checkpointing"] = use_amp- cfg["max_tokens_per_batch"] = 4096- cfg["max_len"] = 1024- cfg["head_max_len"] = 256-- tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "tokenizer"))- model = build_model(cfg, encoder_dir=os.path.join(model_dir, "encoder"))- model.load_state_dict(- load_file(os.path.join(model_dir, "model.safetensors")), strict=True+ config = TrainConfig(+ epochs=args.epochs,+ micro_batch=8,+ grad_accum=1,+ seed=args.seed,+ calib_seed=args.seed,+ max_len=1024,+ head_max_len=256,)- if use_amp:- model.encoder.gradient_checkpointing_enable(- gradient_checkpointing_kwargs={"use_reentrant": False}- )- model.head_checkpointing = True- model.to(device)- model.train()-- all_items = preprocess(tok, cfg, args.data)- if not all_items:- raise SystemExit("No training items produced from --data.")-- order = list(range(len(all_items)))- random.shuffle(order)- n_calib = min(400, len(all_items) // 10)- calib_items = [all_items[i] for i in sorted(order[:n_calib])]- train_items = [all_items[i] for i in sorted(order[n_calib:])]-- micro_batch = 8- group_size = 4- enc_params = [p for n, p in model.named_parameters() if "encoder." in n]- head_params = [p for n, p in model.named_parameters() if "encoder." not in n]- optimizer = torch.optim.AdamW(- [{"params": enc_params, "lr": 2.5e-5}, {"params": head_params, "lr": 1.0e-4}],- weight_decay=0.01,- )- steps_per_epoch = max(1, (len(train_items) + micro_batch - 1) // micro_batch)- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(- optimizer, T_max=steps_per_epoch * args.epochs, eta_min=1e-6- )- scaler = torch.amp.GradScaler("cuda", enabled=True) if use_amp else None-- print(- "Train items: {} ({} held out) | {} epochs | {} steps/epoch".format(- len(train_items), len(calib_items), args.epochs, steps_per_epoch- )- )-- for epoch in range(args.epochs):- random.seed(args.seed + epoch)- random.shuffle(train_items)- sigma = 0.4 + (0.1 - 0.4) * (epoch / max(1, args.epochs - 1))- epoch_loss, n_batches = 0.0, 0- optimizer.zero_grad(set_to_none=True)- for b_idx in range(0, len(train_items), micro_batch):- chunk = train_items[b_idx : b_idx + micro_batch]- if not chunk:- continue- batch = collate(chunk, tok.pad_token_id)- logits, _act = forward(model, batch, device, use_amp)- logits = logits.float()- mask = batch["marker_mask"].to(device)- k = mask.sum(-1, keepdim=True).float()- target = batch["target"].to(device)-- eps = (- torch.randn((group_size,) + logits.shape, device=device) * sigma * mask- )- eps = (eps - eps.sum(-1, keepdim=True) / k) * mask- z = logits.detach().unsqueeze(0) + eps- q = torch.softmax(z.masked_fill(~mask, -1e4), -1)-- with torch.no_grad():- r = proper_reward(- q,- target.unsqueeze(0),- batch["qtype"].to(device),- mask,- w_sph=0.75,- w_rps=1.0,- )- adv = r - r.mean(0, keepdim=True)- adv = adv / (adv.std() + 1e-6)-- logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma**2)- loss = (- -(adv * logp).mean()- - (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1))- .sum(-1)- .mean()- )-- if scaler:- scaler.scale(loss).backward()- scaler.unscale_(optimizer)- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)- scaler.step(optimizer)- scaler.update()- else:- loss.backward()- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)- optimizer.step()- scheduler.step()- optimizer.zero_grad(set_to_none=True)- epoch_loss += loss.item()- n_batches += 1- print(- "Epoch {}/{} | avg loss {:.4f}".format(- epoch + 1, args.epochs, epoch_loss / max(1, n_batches)- ),- flush=True,- )-- model.eval()- calib_preds = []- with torch.no_grad():- for c_idx in range(0, len(calib_items), 16):- c_chunk = calib_items[c_idx : c_idx + 16]- cb = collate(c_chunk, tok.pad_token_id)- l_sub, _ = forward(model, cb, device, use_amp)- l_np = l_sub.float().cpu().numpy()- for r, it in enumerate(c_chunk):- kk = len(it["markers"])- calib_preds.append((it["qtype"], l_np[r, :kk], it["target"]))-- fitted = [1.2, 1.2, 1.2]- for qt in range(3):- sel = [(z, t) for q_type, z, t in calib_preds if q_type == qt]- if sel:- fitted[qt] = fit_one_temp(sel)- print("Fitted temperatures (choice, score, noul):", [round(t, 3) for t in fitted])-- os.makedirs(args.output_dir, exist_ok=True)- save_file(- {key: val.half().contiguous().cpu() for key, val in model.state_dict().items()},- os.path.join(args.output_dir, "model.safetensors"),+ summary = finetune(+ data=args.data,+ model_dir=model_dir,+ output_dir=args.output_dir,+ config=config,+ device=args.device,)- model.encoder.config.save_pretrained(os.path.join(args.output_dir, "encoder"))- tok.save_pretrained(os.path.join(args.output_dir, "tokenizer"))- cfg["fine_tuned"] = True- cfg["temperature"] = fitted- cfg.pop("temperature_by_options", None)- with open(os.path.join(args.output_dir, "rl_agent_config.json"), "w") as f:- json.dump(cfg, f, indent=2)- print("Saved checkpoint to", args.output_dir)+ # Preserve the old wrapper contract verbatim: a qtype that never appeared in the+ # calibration split did not get a fitted 1.0 from the shared fit path; it kept the+ # pre-fit 1.2 no-data default. Low-count present types still report 1.0 via #933.+ if isinstance(summary.get("calibration"), dict):+ for rel in [Path(args.output_dir) / "rl_agent_config.json"]:+ if rel.exists():+ cfg = json.loads(rel.read_text(encoding="utf-8"))+ for name, entry in summary["calibration"].items():+ if isinstance(entry, dict) and entry.get("items") == 0:+ idx = {"choice": 0, "score": 1, "noul": 2}.get(name)+ if idx is not None:+ cfg["temperature"][idx] = 1.2+ cfg.pop("temperature_by_options", None)+ rel.write_text(json.dumps(cfg, indent=2), encoding="utf-8")+ print("Training items: {} ({} held out)".format(+ summary["train_items"], summary["calibration_items"]+ ))+ print("Fitted temperatures (choice, score, noul):", [round(t, 3) for t in summary["temperature"]])+ print("Saved checkpoint to", summary["output_dir"])if __name__ == "__main__":diff --git a/scripts/test_suites.py b/scripts/test_suites.pyindex 512ee675..acf53dba 100644--- a/scripts/test_suites.py+++ b/scripts/test_suites.py@@ -109,6 +109,7 @@ PYTEST_SUITES = ["tests/test_audit_regressions.py","tests/test_truncation_direction.py","tests/test_compile.py",+ "tests/test_finetune_entrypoints.py",]diff --git a/tests/test_finetune_entrypoints.py b/tests/test_finetune_entrypoints.pynew file mode 100644index 00000000..f2cea726--- /dev/null+++ b/tests/test_finetune_entrypoints.py@@ -0,0 +1,153 @@+import argparse+import importlib.util+import json+import sys+from pathlib import Path+from unittest import mock++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))++import research.scripts.finetune_single_device as single # noqa: E402++mps_spec = importlib.util.spec_from_file_location(+ "laya_finetune_typed_decisions_mps",+ ROOT / "notebooks" / "laya_finetune_typed_decisions_mps.py",+)+mps = importlib.util.module_from_spec(mps_spec)+sys.modules["laya_finetune_typed_decisions_mps"] = mps+mps_spec.loader.exec_module(mps)+++def test_single_device_main_delegates_to_laya_train(tmp_path):+ calls = {}+ def fake_finetune(**kwargs):+ calls.update(kwargs)+ return {+ "train_items": 1,+ "calibration_items": 0,+ "temperature": [1.0, 1.0, 1.0],+ "output_dir": str(tmp_path),+ }+ argv = [+ "prog",+ "--data",+ str(tmp_path / "data.jsonl"),+ "--model-dir",+ str(tmp_path / "model"),+ "--output-dir",+ str(tmp_path / "out"),+ "--epochs",+ "2",+ "--seed",+ "7",+ "--device",+ "cpu",+ ]+ with mock.patch.object(single, "finetune", side_effect=fake_finetune):+ with mock.patch.object(sys, "argv", argv):+ single.main()+ assert calls["data"].endswith("data.jsonl")+ assert calls["model_dir"].endswith("model")+ assert calls["output_dir"].endswith("out")+ cfg = calls["config"]+ assert cfg.epochs == 2+ assert cfg.micro_batch == 8+ assert cfg.grad_accum == 1+ assert cfg.max_len == 1024+ assert cfg.head_max_len == 256+ assert calls["device"] == "cpu"+++def test_mps_train_delegates_to_laya_train(tmp_path):+ out = tmp_path / "out"+ out.mkdir()+ (out / "rl_agent_config.json").write_text(json.dumps({"max_len": 1024}), encoding="utf-8")+ calls = {}+ def fake_finetune(**kwargs):+ calls.update(kwargs)+ return {+ "train_items": 1,+ "calibration_items": 0,+ "temperature": [1.0, 1.0, 1.0],+ "output_dir": str(out),+ }+ args = argparse.Namespace(+ epochs=3,+ micro_batch=2,+ grad_accum=16,+ calib_max=400,+ output_dir=str(out),+ no_checkpointing=False,+ device="cpu",+ )+ with mock.patch("laya.train.finetune", side_effect=fake_finetune):+ mps.train(args, str(tmp_path / "model"), str(tmp_path / "rows.jsonl"), None)+ cfg = calls["config"]+ assert cfg.epochs == 3+ assert cfg.micro_batch == 2+ assert cfg.grad_accum == 16+ assert cfg.calib_seed == 20260922+ assert cfg.gradient_checkpointing is True+ assert json.loads((out / "rl_agent_config.json").read_text())["model_name"] == "laya-typed-decisions"+++def test_single_wrapper_zero_vs_low_count_temperature_and_pops_bucket_map(tmp_path):+ out = tmp_path / "out"+ out.mkdir()+ def fake_finetune(**kwargs):+ (out / "rl_agent_config.json").write_text(json.dumps({+ "temperature": [1.0, 1.0, 1.0],+ "temperature_by_options": {"choice:2": 1.3},+ "training": {"laya_train_calibration": {"choice": {"items": 0, "issues": []}}},+ }), encoding="utf-8")+ return {+ "train_items": 1,+ "calibration_items": 0,+ "temperature": [1.0, 1.0, 1.0],+ "output_dir": str(out),+ "calibration": {"choice": {"items": 0, "issues": []}, "score": {"items": 5, "issues": ["not fitted"]}, "noul": {"items": 0, "issues": []}},+ }+ argv = ["prog", "--data", str(tmp_path/"d.jsonl"), "--model-dir", str(tmp_path/"m"), "--output-dir", str(out), "--device", "cpu"]+ with mock.patch.object(single, "finetune", side_effect=fake_finetune):+ with mock.patch.object(sys, "argv", argv):+ single.main()+ cfg = json.loads((out / "rl_agent_config.json").read_text())+ assert cfg["temperature"] == [1.2, 1.0, 1.2]+ assert cfg["temperature_by_options"] if "temperature_by_options" in cfg else True+ assert "temperature_by_options" not in cfg+ assert cfg["training"]["laya_train_calibration"]["choice"]["items"] == 0+++def test_mps_train_wrapper_config_parity_and_bucket_map_removed(tmp_path):+ out = tmp_path / "out"+ out.mkdir()+ ckpt = out / "checkpoint_latest"+ ckpt.mkdir()+ def make_cfg():+ return {"fine_tuned": True, "temperature": [1.0, 1.0, 1.0], "temperature_by_options": {"score:4": 1.1}, "training": {"laya_train_calibration": {"score": {"items": 0, "issues": []}}}}+ (out / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")+ (ckpt / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")+ def fake_finetune(**kwargs):+ (out / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")+ (ckpt / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")+ return {"train_items": 1, "calibration_items": 0, "temperature": [1.0, 1.0, 1.0], "output_dir": str(out), "calibration": {"choice": {"items": 0}, "score": {"items": 0}, "noul": {"items": 12}}}+ args = argparse.Namespace(epochs=1, micro_batch=2, grad_accum=16, calib_max=400, output_dir=str(out), no_checkpointing=False, device="cpu")+ with mock.patch("laya.train.finetune", side_effect=fake_finetune):+ mps.train(args, str(tmp_path/"model"), str(tmp_path/"rows.jsonl"), None)+ for path in [out / "rl_agent_config.json", ckpt / "rl_agent_config.json"]:+ cfg = json.loads(path.read_text())+ assert cfg["temperature"] == [1.2, 1.2, 1.0]+ assert "temperature_by_options" not in cfg+ assert cfg["training"]["laya_train_calibration"]["score"]["items"] == 0+ assert cfg["model_name"] == "laya-typed-decisions"+ assert cfg["max_tokens_per_batch"] == 2048+++def test_no_duplicated_fitter_or_training_loop_remains():+ single_text = (ROOT / "research" / "scripts" / "finetune_single_device.py").read_text(encoding="utf-8")+ mps_text = (ROOT / "notebooks" / "laya_finetune_typed_decisions_mps.py").read_text(encoding="utf-8")+ assert "fit_one_temp" not in single_text+ assert "fit_temperature(" not in mps_text+ assert "def preprocess" not in single_text+ assert "while n_batches % args.grad_accum" not in mps_text