Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

refactor(finetune): converge single-process wrappers on laya.train

Bruce-Yiicebf8b16a5bc
diff --git a/notebooks/laya_finetune_typed_decisions_mps.py b/notebooks/laya_finetune_typed_decisions_mps.py
index 6f854be3..17ab53f1 100644
--- a/notebooks/laya_finetune_typed_decisions_mps.py
+++ b/notebooks/laya_finetune_typed_decisions_mps.py
@@ -16,17 +16,12 @@ Useful examples:
import argparse
import gc
import json
-import math
-import random
from pathlib import Path
import torch
from huggingface_hub import snapshot_download
-from safetensors.torch import load_file, save_file
-from transformers import AutoTokenizer
from laya.agent import _fix_tokenizer_config
-from laya.common import QTYPES, TEMP_MAX, TEMP_MIN, build_model, build_sequence, proper_reward, render_options
MODEL_ID = "convaiinnovations/laya"
DATASET_ID = "LocalLLaMA/typed-decisions"
@@ -55,316 +50,97 @@ def prepare_model(model_dir):
return str(model_dir)
-def build_training_item(tokenizer, cfg, state, question, gold_question):
- qtype = question["type"]
- criteria = question.get("criteria", {})
-
- if qtype == "choice":
- keys = list(criteria.keys())
- target = [gold_question["probabilities"].get(k, 0.0) for k in keys]
- elif qtype == "noul":
- target = [
- gold_question["probabilities"].get("false", 0.5),
- gold_question["probabilities"].get("true", 0.5),
- ]
- elif qtype == "score":
- n_levels = len(criteria) if isinstance(criteria, list) else 4
- target = [gold_question["probabilities"].get(str(i), 0.0) for i in range(n_levels)]
- else:
- return None
-
- total = sum(target)
- if total > 0:
- target = [float(x) / total for x in target]
- else:
- target = [1.0 / len(target)] * len(target)
-
- label = target.index(max(target))
- n_options = len(render_options({"t": qtype, "crit": criteria}))
- sequence, markers = build_sequence(
- tokenizer,
- state,
- {"t": qtype, "ins": question["instructions"], "crit": criteria},
- cfg["max_len"],
- cfg["head_max_len"],
- )
- if len(markers) != n_options:
- return None
-
- return {
- "ids": sequence,
- "markers": markers,
- "qtype": QTYPES[qtype],
- "target": target,
- "label": label,
- }
-
-
def prepare_items(model_dir, items_path, force=False):
+ # The convergence target is `laya.train`; its input is row-level JSONL, not
+ # pre-tokenized item tensors. Keep the cache honest: when the requested cache
+ # changes, also invalidate the row cache.
items_path = Path(items_path)
+ rows_path = items_path.with_suffix(items_path.suffix + '.rows.jsonl')
model_path = Path(model_dir).resolve()
- with open(model_path / "rl_agent_config.json") as f:
+ with open(model_path / 'rl_agent_config.json') as f:
cfg = json.load(f)
- max_len = cfg.get("max_len", 1024)
- head_max_len = cfg.get("head_max_len", 256)
- cache_meta_path = items_path.with_name(items_path.name + ".meta.json")
+ max_len = cfg.get('max_len', 1024)
+ head_max_len = cfg.get('head_max_len', 256)
+ cache_meta_path = rows_path.with_name(rows_path.name + '.meta.json')
cache_key = {
- "model_dir": str(model_path),
- "max_len": max_len,
- "head_max_len": head_max_len,
- "dataset": DATASET_ID,
- "split": "train",
+ 'model_dir': str(model_path),
+ 'max_len': max_len,
+ 'head_max_len': head_max_len,
+ 'dataset': DATASET_ID,
+ 'split': 'train',
+ 'format': 'rows_jsonl',
}
- if items_path.exists() and not force:
+ if rows_path.exists() and not force:
if not cache_meta_path.exists():
- # Legacy item files predate the sidecar metadata. They remain
- # usable offline; newly written caches always receive a key.
- print(f"Using legacy cached training items without metadata: {items_path}")
- return
+ print(f'Using legacy cached training rows without metadata: {rows_path}')
+ return str(rows_path)
try:
with open(cache_meta_path) as f:
cached_key = json.load(f)
except (OSError, json.JSONDecodeError):
cached_key = None
if cached_key == cache_key:
- print(f"Using cached training items: {items_path}")
- return
- print("Training-item cache key changed; rebuilding the cache.")
+ print(f'Using cached training rows: {rows_path}')
+ return str(rows_path)
+ print('Training-row cache key changed; rebuilding the cache.')
- # Keep datasets optional when a compatible local cache is already available.
from datasets import load_dataset
- tokenizer = AutoTokenizer.from_pretrained(model_path / "tokenizer")
- print(f"Downloading dataset {DATASET_ID} ...")
- dataset = load_dataset(DATASET_ID, "all", split="train")
-
- items = []
- skipped = 0
- for row in dataset:
- state = json.loads(row["state"])
- questions = json.loads(row["questions"])
- gold = json.loads(row["gold"])
- for qid, question in questions.items():
- if qid not in gold:
- continue
- item = build_training_item(
- tokenizer,
- {**cfg, "max_len": max_len, "head_max_len": head_max_len},
- state,
- question,
- gold[qid],
- )
- if item is None:
- skipped += 1
- else:
- items.append(item)
-
- items_path.parent.mkdir(parents=True, exist_ok=True)
- torch.save(items, items_path)
- with open(cache_meta_path, "w") as f:
+ print(f'Downloading dataset {DATASET_ID} ...')
+ dataset = load_dataset(DATASET_ID, 'all', split='train')
+ rows_path.parent.mkdir(parents=True, exist_ok=True)
+ n_rows = 0
+ with open(rows_path, 'w', encoding='utf-8') as f:
+ for row in dataset:
+ state = json.loads(row['state'])
+ questions = json.loads(row['questions'])
+ gold = json.loads(row['gold'])
+ f.write(json.dumps({'state': state, 'questions': questions, 'gold': gold}, ensure_ascii=False) + '\n')
+ n_rows += 1
+ with open(cache_meta_path, 'w') as f:
json.dump(cache_key, f, indent=2)
- print(f"Saved {len(items)} training items to {items_path}; skipped={skipped}")
-
-
-def collate(items, pad_id):
- batch_size = len(items)
- seq_len = max(len(item["ids"]) for item in items)
- kmax = max(len(item["markers"]) for item in items)
- input_ids = torch.full((batch_size, seq_len), pad_id, dtype=torch.long)
- attention = torch.zeros((batch_size, seq_len), dtype=torch.long)
- marker_pos = torch.zeros((batch_size, kmax), dtype=torch.long)
- marker_mask = torch.zeros((batch_size, kmax), dtype=torch.bool)
- target = torch.zeros((batch_size, kmax), dtype=torch.float32)
-
- for i, item in enumerate(items):
- length = len(item["ids"])
- input_ids[i, :length] = torch.tensor(item["ids"], dtype=torch.long)
- attention[i, :length] = 1
- k = len(item["markers"])
- marker_pos[i, :k] = torch.tensor(item["markers"], dtype=torch.long)
- marker_mask[i, :k] = True
- target[i, :len(item["target"])] = torch.tensor(item["target"], dtype=torch.float32)
-
- return (
- input_ids,
- attention,
- marker_pos,
- marker_mask,
- target,
- torch.tensor([item["qtype"] for item in items], dtype=torch.long),
- )
-
-
-def fit_temperature(samples):
- if len(samples) < 10:
- return 1.0
- kmax = max(len(logits) for logits, _ in samples)
- logits = torch.full((len(samples), kmax), -1e4)
- targets = torch.zeros((len(samples), kmax))
- for i, (values, target) in enumerate(samples):
- logits[i, :len(values)] = torch.as_tensor(values)
- targets[i, :len(target)] = torch.as_tensor(target, dtype=torch.float32)
-
- log_temperature = torch.zeros(1, requires_grad=True)
- optimizer = torch.optim.LBFGS([log_temperature], lr=0.1, max_iter=100)
-
- def closure():
- optimizer.zero_grad()
- loss = -(targets * torch.log_softmax(logits / log_temperature.exp(), -1)).sum(-1).mean()
- loss.backward()
- return loss
-
- optimizer.step(closure)
- return float(torch.clamp(log_temperature.exp(), TEMP_MIN, TEMP_MAX).item())
-
-
-def save_checkpoint(model, tokenizer, cfg, output_dir, epoch, final=False):
- path = Path(output_dir) if final else Path(output_dir) / "checkpoint_latest"
- path.mkdir(parents=True, exist_ok=True)
- weights = {
- name: value.detach().half().cpu().contiguous()
- for name, value in model.state_dict().items()
- }
- save_file(weights, str(path / "model.safetensors"))
- model.encoder.config.save_pretrained(path / "encoder")
- tokenizer.save_pretrained(path / "tokenizer")
- with open(path / "checkpoint_meta.json", "w") as f:
- json.dump({"epoch": epoch, "final": final}, f, indent=2)
- with open(path / "rl_agent_config.json", "w") as f:
- json.dump(cfg, f, indent=2)
-
-
-def train(args, model_dir, items_path, device):
- with open(Path(model_dir) / "rl_agent_config.json") as f:
- cfg = json.load(f)
- cfg.update({"max_tokens_per_batch": 2048, "max_len": 1024, "head_max_len": 256})
- if not args.no_checkpointing:
- cfg["gradient_checkpointing"] = True
-
- tokenizer = AutoTokenizer.from_pretrained(Path(model_dir) / "tokenizer")
- model = build_model(cfg, encoder_dir=Path(model_dir) / "encoder")
- model.load_state_dict(load_file(str(Path(model_dir) / "model.safetensors")), strict=True)
- model.float()
-
- if not args.no_checkpointing:
- model.encoder.gradient_checkpointing_enable(
- gradient_checkpointing_kwargs={"use_reentrant": False}
- )
- model.head_checkpointing = True
- model.to(device).train()
-
- all_items = torch.load(items_path, map_location="cpu", weights_only=False)
- order = list(range(len(all_items)))
- random.Random(20260922).shuffle(order)
- n_calib = min(args.calib_max, len(all_items) // 10)
- calib_items = [all_items[i] for i in sorted(order[:n_calib])]
- train_items = [all_items[i] for i in sorted(order[n_calib:])]
-
- encoder_params = [p for n, p in model.named_parameters() if "encoder." in n]
- head_params = [p for n, p in model.named_parameters() if "encoder." not in n]
- optimizer = torch.optim.AdamW(
- [{"params": encoder_params, "lr": 2.5e-5}, {"params": head_params, "lr": 1e-4}],
- weight_decay=0.01,
+ print(f'Saved {n_rows} training rows to {rows_path}')
+ return str(rows_path)
+
+
+def train(args, model_dir, rows_path, device):
+ from laya.train import TrainConfig, finetune
+
+ config = TrainConfig(
+ epochs=args.epochs,
+ micro_batch=args.micro_batch,
+ grad_accum=args.grad_accum,
+ calib_max=args.calib_max,
+ calib_seed=20260922,
+ seed=42,
+ max_len=1024,
+ head_max_len=256,
+ gradient_checkpointing=not args.no_checkpointing,
+ amp=False,
)
- updates = max(1, math.ceil(len(train_items) / args.micro_batch / args.grad_accum) * args.epochs)
- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
- optimizer, T_max=updates, eta_min=1e-6
+ summary = finetune(
+ data=str(rows_path),
+ model_dir=str(model_dir),
+ output_dir=str(args.output_dir),
+ config=config,
+ device=args.device if args.device != 'auto' else ('mps' if torch.backends.mps.is_available() else 'cpu'),
)
-
- print(f"Device: {device}")
- print(f"Training items: {len(train_items)}; calibration items: {len(calib_items)}")
- print(f"micro_batch={args.micro_batch}; grad_accum={args.grad_accum}; epochs={args.epochs}")
-
- for epoch in range(args.epochs):
- random.Random(42 + epoch).shuffle(train_items)
- optimizer.zero_grad(set_to_none=True)
- total_loss = 0.0
- n_batches = 0
- sigma = 0.4 + (0.1 - 0.4) * epoch / max(1, args.epochs - 1)
-
- for start in range(0, len(train_items), args.micro_batch):
- chunk = train_items[start:start + args.micro_batch]
- ids, attention, positions, mask, target, qtype = collate(chunk, tokenizer.pad_token_id)
- ids, attention = ids.to(device), attention.to(device)
- positions = positions.to(device)
- mask = mask.to(device)
- target = target.to(device)
- qtype = qtype.to(device)
-
- logits, activation = model(ids, attention, positions, mask, qtype)
- logits = logits.float()
- k = mask.sum(-1, keepdim=True).float()
-
- eps = torch.randn((4,) + logits.shape, device=device) * sigma * mask
- eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
- noisy_logits = logits.detach().unsqueeze(0) + eps
- probabilities = torch.softmax(noisy_logits.masked_fill(~mask, -1e4), -1)
-
- with torch.no_grad():
- reward = proper_reward(
- probabilities,
- target.unsqueeze(0),
- qtype,
- mask,
- w_sph=0.75,
- w_rps=1.0,
- )
- advantage = reward - reward.mean(0, keepdim=True)
- advantage = advantage / (advantage.std() + 1e-6)
-
- logp = -(((noisy_logits - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma**2)
- loss_rl = -(advantage * logp).mean()
- loss_ce = -(
- target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)
- ).sum(-1).mean()
- loss = (loss_rl + loss_ce + 0.0 * activation.sum()) / args.grad_accum
- loss.backward()
-
- n_batches += 1
- if n_batches % args.grad_accum == 0 or start + args.micro_batch >= len(train_items):
- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
- optimizer.step()
- scheduler.step()
- optimizer.zero_grad(set_to_none=True)
-
- total_loss += loss.item() * args.grad_accum
- if n_batches % 100 == 0:
- print(f"epoch {epoch + 1}/{args.epochs}, step {n_batches}, loss={loss.item() * args.grad_accum:.4f}")
-
- avg_loss = total_loss / max(1, n_batches)
- print(f"Epoch {epoch + 1}/{args.epochs} complete; avg_loss={avg_loss:.4f}")
- save_checkpoint(model, tokenizer, cfg, args.output_dir, epoch + 1)
-
- print("Running temperature calibration ...")
- model.eval()
- samples = [[] for _ in range(3)]
- with torch.no_grad():
- for start in range(0, len(calib_items), args.micro_batch):
- chunk = calib_items[start:start + args.micro_batch]
- ids, attention, positions, mask, target, qtype = collate(chunk, tokenizer.pad_token_id)
- logits, _ = model(
- ids.to(device), attention.to(device), positions.to(device), mask.to(device), qtype.to(device)
- )
- for i, item in enumerate(chunk):
- qtype_id = item["qtype"]
- samples[qtype_id].append(
- (logits[i, :len(item["markers"])].cpu(), item["target"])
- )
-
- temperatures = [fit_temperature(group) if group else 1.2 for group in samples]
- save_checkpoint(model, tokenizer, cfg, args.output_dir, args.epochs, final=True)
- cfg.update({
- "fine_tuned": True,
- "model_name": "laya-typed-decisions",
- "temperature": temperatures,
- })
- cfg.pop("temperature_by_options", None)
- with open(Path(args.output_dir) / "rl_agent_config.json", "w") as f:
- json.dump(cfg, f, indent=2)
- with open(Path(args.output_dir) / "checkpoint_latest" / "rl_agent_config.json", "w") as f:
- json.dump(cfg, f, indent=2)
- print(f"Model saved to {args.output_dir}")
- print(f"Temperatures: {temperatures}")
+ # Wrapper-visible metadata and persistence preserved from the old entry point.
+ for rel in [Path(args.output_dir) / 'rl_agent_config.json', Path(args.output_dir) / 'checkpoint_latest' / 'rl_agent_config.json']:
+ if rel.exists():
+ cfg = json.loads(rel.read_text(encoding='utf-8'))
+ for name, entry in (summary.get('calibration') or {}).items():
+ if isinstance(entry, dict) and entry.get('items') == 0:
+ idx = {'choice': 0, 'score': 1, 'noul': 2}.get(name)
+ if idx is not None:
+ cfg['temperature'][idx] = 1.2
+ cfg.pop('temperature_by_options', None)
+ cfg['model_name'] = 'laya-typed-decisions'
+ cfg['max_tokens_per_batch'] = 2048
+ rel.write_text(json.dumps(cfg, indent=2), encoding='utf-8')
+ print(f'Train items: {summary["train_items"]}; calibration items: {summary["calibration_items"]}')
+ print('Fitted temperatures (choice, score, noul):', [round(t, 3) for t in summary['temperature']])
+ print(f'Model saved to {args.output_dir}')
def main():
@@ -395,8 +171,8 @@ def main():
torch.set_float32_matmul_precision("high")
device = choose_device(args.device)
model_dir = prepare_model(args.model_dir)
- prepare_items(model_dir, args.items, force=args.force_preprocess)
- train(args, model_dir, args.items, device)
+ rows_path = prepare_items(model_dir, args.items, force=args.force_preprocess)
+ train(args, model_dir, rows_path, device)
gc.collect()
diff --git a/research/scripts/finetune_single_device.py b/research/scripts/finetune_single_device.py
index fb5af748..ce842618 100644
--- a/research/scripts/finetune_single_device.py
+++ b/research/scripts/finetune_single_device.py
@@ -17,143 +17,16 @@ Usage:
import argparse
import json
import os
-import random
import sys
+from pathlib import Path
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
sys.path.insert(0, ROOT)
-import torch
from huggingface_hub import snapshot_download
-from safetensors.torch import load_file, save_file
-from transformers import AutoTokenizer
from laya.agent import _fix_tokenizer_config
-from laya.common import (
- QTYPES,
- TEMP_MAX,
- TEMP_MIN,
- build_model,
- build_sequence,
- proper_reward,
- render_options,
-)
-
-
-def build_training_item(tok, cfg, state, q, gold_q):
- t = q["type"]
- crit = q.get("criteria", {})
- if t == "choice":
- keys = list(crit.keys())
- target = [gold_q["probabilities"].get(k, 0.0) for k in keys]
- elif t == "noul":
- target = [
- gold_q["probabilities"].get("false", 0.5),
- gold_q["probabilities"].get("true", 0.5),
- ]
- else:
- n_levels = len(crit) if isinstance(crit, list) else 4
- target = [gold_q["probabilities"].get(str(i), 0.0) for i in range(n_levels)]
- total = sum(target)
- target = (
- [v / total for v in target] if total > 0 else [1.0 / len(target)] * len(target)
- )
- seq, markers = build_sequence(
- tok,
- state,
- {"t": t, "ins": q["instructions"], "crit": crit},
- cfg["max_len"],
- cfg["head_max_len"],
- )
- if len(markers) != len(render_options({"t": t, "crit": crit})):
- return None
- return {
- "ids": seq,
- "markers": markers,
- "qtype": QTYPES[t],
- "target": target,
- "label": target.index(max(target)),
- }
-
-
-def preprocess(tok, cfg, data_path):
- items = []
- with open(data_path, encoding="utf-8") as f:
- for line in f:
- row = json.loads(line)
- for qid, q in row["questions"].items():
- if qid not in row["gold"]:
- continue
- item = build_training_item(tok, cfg, row["state"], q, row["gold"][qid])
- if item is not None:
- items.append(item)
- return items
-
-
-def collate(items, pad_id):
- n, length = len(items), max(len(it["ids"]) for it in items)
- kmax = max(len(it["markers"]) for it in items)
- ids = torch.full((n, length), pad_id, dtype=torch.long)
- att = torch.zeros((n, length), dtype=torch.long)
- mpos = torch.zeros((n, kmax), dtype=torch.long)
- mmask = torch.zeros((n, kmax), dtype=torch.bool)
- target = torch.zeros((n, kmax), dtype=torch.float32)
- for i, it in enumerate(items):
- ids[i, : len(it["ids"])] = torch.tensor(it["ids"])
- att[i, : len(it["ids"])] = 1
- k = len(it["markers"])
- mpos[i, :k] = torch.tensor(it["markers"])
- mmask[i, :k] = True
- target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)
- return {
- "input_ids": ids,
- "attention_mask": att,
- "marker_pos": mpos,
- "marker_mask": mmask,
- "target": target,
- "qtype": torch.tensor([it["qtype"] for it in items]),
- }
-
-
-def forward(model, batch, device, use_amp):
- if use_amp:
- with torch.autocast("cuda", dtype=torch.float16):
- return model(
- batch["input_ids"].to(device),
- batch["attention_mask"].to(device),
- batch["marker_pos"].to(device),
- batch["marker_mask"].to(device),
- batch["qtype"].to(device),
- )
- return model(
- batch["input_ids"].to(device),
- batch["attention_mask"].to(device),
- batch["marker_pos"].to(device),
- batch["marker_mask"].to(device),
- batch["qtype"].to(device),
- )
-
-
-def fit_one_temp(sel):
- if len(sel) < 10:
- return 1.0
- kmax = max(len(z) for z, _ in sel)
- Z = torch.full((len(sel), kmax), -1e4)
- T = torch.zeros((len(sel), kmax))
- for i, (z, t) in enumerate(sel):
- Z[i, : len(z)] = torch.tensor(z)
- T[i, : len(t)] = torch.tensor(t, dtype=torch.float32)
- log_t = torch.zeros(1, requires_grad=True)
- opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
-
- def closure():
- opt.zero_grad()
- loss = -(T * torch.log_softmax(Z / log_t.exp(), -1)).sum(-1).mean()
- loss.backward()
- return loss
-
- opt.step(closure)
- return float(torch.clamp(log_t.exp(), TEMP_MIN, TEMP_MAX).item())
+from laya.train import TrainConfig, finetune
def main():
@@ -176,20 +49,6 @@ def main():
parser.add_argument("--seed", type=int, default=0)
args = parser.parse_args()
- if args.device == "auto":
- device = (
- torch.device("cuda", 0)
- if torch.cuda.is_available()
- else torch.device("cpu")
- )
- else:
- device = torch.device(args.device)
- use_amp = device.type == "cuda"
- torch.manual_seed(args.seed)
- random.seed(args.seed)
- torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", os.cpu_count() or 1)))
- print("Device:", device)
-
if args.model_dir:
model_dir = args.model_dir
else:
@@ -202,153 +61,41 @@ def main():
)
_fix_tokenizer_config(model_dir)
- with open(os.path.join(model_dir, "rl_agent_config.json")) as f:
- cfg = json.load(f)
- cfg["gradient_checkpointing"] = use_amp
- cfg["max_tokens_per_batch"] = 4096
- cfg["max_len"] = 1024
- cfg["head_max_len"] = 256
-
- tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "tokenizer"))
- model = build_model(cfg, encoder_dir=os.path.join(model_dir, "encoder"))
- model.load_state_dict(
- load_file(os.path.join(model_dir, "model.safetensors")), strict=True
+ config = TrainConfig(
+ epochs=args.epochs,
+ micro_batch=8,
+ grad_accum=1,
+ seed=args.seed,
+ calib_seed=args.seed,
+ max_len=1024,
+ head_max_len=256,
)
- if use_amp:
- model.encoder.gradient_checkpointing_enable(
- gradient_checkpointing_kwargs={"use_reentrant": False}
- )
- model.head_checkpointing = True
- model.to(device)
- model.train()
-
- all_items = preprocess(tok, cfg, args.data)
- if not all_items:
- raise SystemExit("No training items produced from --data.")
-
- order = list(range(len(all_items)))
- random.shuffle(order)
- n_calib = min(400, len(all_items) // 10)
- calib_items = [all_items[i] for i in sorted(order[:n_calib])]
- train_items = [all_items[i] for i in sorted(order[n_calib:])]
-
- micro_batch = 8
- group_size = 4
- enc_params = [p for n, p in model.named_parameters() if "encoder." in n]
- head_params = [p for n, p in model.named_parameters() if "encoder." not in n]
- optimizer = torch.optim.AdamW(
- [{"params": enc_params, "lr": 2.5e-5}, {"params": head_params, "lr": 1.0e-4}],
- weight_decay=0.01,
- )
- steps_per_epoch = max(1, (len(train_items) + micro_batch - 1) // micro_batch)
- scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
- optimizer, T_max=steps_per_epoch * args.epochs, eta_min=1e-6
- )
- scaler = torch.amp.GradScaler("cuda", enabled=True) if use_amp else None
-
- print(
- "Train items: {} ({} held out) | {} epochs | {} steps/epoch".format(
- len(train_items), len(calib_items), args.epochs, steps_per_epoch
- )
- )
-
- for epoch in range(args.epochs):
- random.seed(args.seed + epoch)
- random.shuffle(train_items)
- sigma = 0.4 + (0.1 - 0.4) * (epoch / max(1, args.epochs - 1))
- epoch_loss, n_batches = 0.0, 0
- optimizer.zero_grad(set_to_none=True)
- for b_idx in range(0, len(train_items), micro_batch):
- chunk = train_items[b_idx : b_idx + micro_batch]
- if not chunk:
- continue
- batch = collate(chunk, tok.pad_token_id)
- logits, _act = forward(model, batch, device, use_amp)
- logits = logits.float()
- mask = batch["marker_mask"].to(device)
- k = mask.sum(-1, keepdim=True).float()
- target = batch["target"].to(device)
-
- eps = (
- torch.randn((group_size,) + logits.shape, device=device) * sigma * mask
- )
- eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
- z = logits.detach().unsqueeze(0) + eps
- q = torch.softmax(z.masked_fill(~mask, -1e4), -1)
-
- with torch.no_grad():
- r = proper_reward(
- q,
- target.unsqueeze(0),
- batch["qtype"].to(device),
- mask,
- w_sph=0.75,
- w_rps=1.0,
- )
- adv = r - r.mean(0, keepdim=True)
- adv = adv / (adv.std() + 1e-6)
-
- logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma**2)
- loss = (
- -(adv * logp).mean()
- - (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1))
- .sum(-1)
- .mean()
- )
-
- if scaler:
- scaler.scale(loss).backward()
- scaler.unscale_(optimizer)
- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
- scaler.step(optimizer)
- scaler.update()
- else:
- loss.backward()
- torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
- optimizer.step()
- scheduler.step()
- optimizer.zero_grad(set_to_none=True)
- epoch_loss += loss.item()
- n_batches += 1
- print(
- "Epoch {}/{} | avg loss {:.4f}".format(
- epoch + 1, args.epochs, epoch_loss / max(1, n_batches)
- ),
- flush=True,
- )
-
- model.eval()
- calib_preds = []
- with torch.no_grad():
- for c_idx in range(0, len(calib_items), 16):
- c_chunk = calib_items[c_idx : c_idx + 16]
- cb = collate(c_chunk, tok.pad_token_id)
- l_sub, _ = forward(model, cb, device, use_amp)
- l_np = l_sub.float().cpu().numpy()
- for r, it in enumerate(c_chunk):
- kk = len(it["markers"])
- calib_preds.append((it["qtype"], l_np[r, :kk], it["target"]))
-
- fitted = [1.2, 1.2, 1.2]
- for qt in range(3):
- sel = [(z, t) for q_type, z, t in calib_preds if q_type == qt]
- if sel:
- fitted[qt] = fit_one_temp(sel)
- print("Fitted temperatures (choice, score, noul):", [round(t, 3) for t in fitted])
-
- os.makedirs(args.output_dir, exist_ok=True)
- save_file(
- {key: val.half().contiguous().cpu() for key, val in model.state_dict().items()},
- os.path.join(args.output_dir, "model.safetensors"),
+ summary = finetune(
+ data=args.data,
+ model_dir=model_dir,
+ output_dir=args.output_dir,
+ config=config,
+ device=args.device,
)
- model.encoder.config.save_pretrained(os.path.join(args.output_dir, "encoder"))
- tok.save_pretrained(os.path.join(args.output_dir, "tokenizer"))
- cfg["fine_tuned"] = True
- cfg["temperature"] = fitted
- cfg.pop("temperature_by_options", None)
- with open(os.path.join(args.output_dir, "rl_agent_config.json"), "w") as f:
- json.dump(cfg, f, indent=2)
- print("Saved checkpoint to", args.output_dir)
+ # Preserve the old wrapper contract verbatim: a qtype that never appeared in the
+ # calibration split did not get a fitted 1.0 from the shared fit path; it kept the
+ # pre-fit 1.2 no-data default. Low-count present types still report 1.0 via #933.
+ if isinstance(summary.get("calibration"), dict):
+ for rel in [Path(args.output_dir) / "rl_agent_config.json"]:
+ if rel.exists():
+ cfg = json.loads(rel.read_text(encoding="utf-8"))
+ for name, entry in summary["calibration"].items():
+ if isinstance(entry, dict) and entry.get("items") == 0:
+ idx = {"choice": 0, "score": 1, "noul": 2}.get(name)
+ if idx is not None:
+ cfg["temperature"][idx] = 1.2
+ cfg.pop("temperature_by_options", None)
+ rel.write_text(json.dumps(cfg, indent=2), encoding="utf-8")
+ print("Training items: {} ({} held out)".format(
+ summary["train_items"], summary["calibration_items"]
+ ))
+ print("Fitted temperatures (choice, score, noul):", [round(t, 3) for t in summary["temperature"]])
+ print("Saved checkpoint to", summary["output_dir"])
if __name__ == "__main__":
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index 512ee675..acf53dba 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -109,6 +109,7 @@ PYTEST_SUITES = [
"tests/test_audit_regressions.py",
"tests/test_truncation_direction.py",
"tests/test_compile.py",
+ "tests/test_finetune_entrypoints.py",
]
diff --git a/tests/test_finetune_entrypoints.py b/tests/test_finetune_entrypoints.py
new file mode 100644
index 00000000..f2cea726
--- /dev/null
+++ b/tests/test_finetune_entrypoints.py
@@ -0,0 +1,153 @@
+import argparse
+import importlib.util
+import json
+import sys
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+import research.scripts.finetune_single_device as single # noqa: E402
+
+mps_spec = importlib.util.spec_from_file_location(
+ "laya_finetune_typed_decisions_mps",
+ ROOT / "notebooks" / "laya_finetune_typed_decisions_mps.py",
+)
+mps = importlib.util.module_from_spec(mps_spec)
+sys.modules["laya_finetune_typed_decisions_mps"] = mps
+mps_spec.loader.exec_module(mps)
+
+
+def test_single_device_main_delegates_to_laya_train(tmp_path):
+ calls = {}
+ def fake_finetune(**kwargs):
+ calls.update(kwargs)
+ return {
+ "train_items": 1,
+ "calibration_items": 0,
+ "temperature": [1.0, 1.0, 1.0],
+ "output_dir": str(tmp_path),
+ }
+ argv = [
+ "prog",
+ "--data",
+ str(tmp_path / "data.jsonl"),
+ "--model-dir",
+ str(tmp_path / "model"),
+ "--output-dir",
+ str(tmp_path / "out"),
+ "--epochs",
+ "2",
+ "--seed",
+ "7",
+ "--device",
+ "cpu",
+ ]
+ with mock.patch.object(single, "finetune", side_effect=fake_finetune):
+ with mock.patch.object(sys, "argv", argv):
+ single.main()
+ assert calls["data"].endswith("data.jsonl")
+ assert calls["model_dir"].endswith("model")
+ assert calls["output_dir"].endswith("out")
+ cfg = calls["config"]
+ assert cfg.epochs == 2
+ assert cfg.micro_batch == 8
+ assert cfg.grad_accum == 1
+ assert cfg.max_len == 1024
+ assert cfg.head_max_len == 256
+ assert calls["device"] == "cpu"
+
+
+def test_mps_train_delegates_to_laya_train(tmp_path):
+ out = tmp_path / "out"
+ out.mkdir()
+ (out / "rl_agent_config.json").write_text(json.dumps({"max_len": 1024}), encoding="utf-8")
+ calls = {}
+ def fake_finetune(**kwargs):
+ calls.update(kwargs)
+ return {
+ "train_items": 1,
+ "calibration_items": 0,
+ "temperature": [1.0, 1.0, 1.0],
+ "output_dir": str(out),
+ }
+ args = argparse.Namespace(
+ epochs=3,
+ micro_batch=2,
+ grad_accum=16,
+ calib_max=400,
+ output_dir=str(out),
+ no_checkpointing=False,
+ device="cpu",
+ )
+ with mock.patch("laya.train.finetune", side_effect=fake_finetune):
+ mps.train(args, str(tmp_path / "model"), str(tmp_path / "rows.jsonl"), None)
+ cfg = calls["config"]
+ assert cfg.epochs == 3
+ assert cfg.micro_batch == 2
+ assert cfg.grad_accum == 16
+ assert cfg.calib_seed == 20260922
+ assert cfg.gradient_checkpointing is True
+ assert json.loads((out / "rl_agent_config.json").read_text())["model_name"] == "laya-typed-decisions"
+
+
+def test_single_wrapper_zero_vs_low_count_temperature_and_pops_bucket_map(tmp_path):
+ out = tmp_path / "out"
+ out.mkdir()
+ def fake_finetune(**kwargs):
+ (out / "rl_agent_config.json").write_text(json.dumps({
+ "temperature": [1.0, 1.0, 1.0],
+ "temperature_by_options": {"choice:2": 1.3},
+ "training": {"laya_train_calibration": {"choice": {"items": 0, "issues": []}}},
+ }), encoding="utf-8")
+ return {
+ "train_items": 1,
+ "calibration_items": 0,
+ "temperature": [1.0, 1.0, 1.0],
+ "output_dir": str(out),
+ "calibration": {"choice": {"items": 0, "issues": []}, "score": {"items": 5, "issues": ["not fitted"]}, "noul": {"items": 0, "issues": []}},
+ }
+ argv = ["prog", "--data", str(tmp_path/"d.jsonl"), "--model-dir", str(tmp_path/"m"), "--output-dir", str(out), "--device", "cpu"]
+ with mock.patch.object(single, "finetune", side_effect=fake_finetune):
+ with mock.patch.object(sys, "argv", argv):
+ single.main()
+ cfg = json.loads((out / "rl_agent_config.json").read_text())
+ assert cfg["temperature"] == [1.2, 1.0, 1.2]
+ assert cfg["temperature_by_options"] if "temperature_by_options" in cfg else True
+ assert "temperature_by_options" not in cfg
+ assert cfg["training"]["laya_train_calibration"]["choice"]["items"] == 0
+
+
+def test_mps_train_wrapper_config_parity_and_bucket_map_removed(tmp_path):
+ out = tmp_path / "out"
+ out.mkdir()
+ ckpt = out / "checkpoint_latest"
+ ckpt.mkdir()
+ def make_cfg():
+ return {"fine_tuned": True, "temperature": [1.0, 1.0, 1.0], "temperature_by_options": {"score:4": 1.1}, "training": {"laya_train_calibration": {"score": {"items": 0, "issues": []}}}}
+ (out / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")
+ (ckpt / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")
+ def fake_finetune(**kwargs):
+ (out / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")
+ (ckpt / "rl_agent_config.json").write_text(json.dumps(make_cfg()), encoding="utf-8")
+ return {"train_items": 1, "calibration_items": 0, "temperature": [1.0, 1.0, 1.0], "output_dir": str(out), "calibration": {"choice": {"items": 0}, "score": {"items": 0}, "noul": {"items": 12}}}
+ args = argparse.Namespace(epochs=1, micro_batch=2, grad_accum=16, calib_max=400, output_dir=str(out), no_checkpointing=False, device="cpu")
+ with mock.patch("laya.train.finetune", side_effect=fake_finetune):
+ mps.train(args, str(tmp_path/"model"), str(tmp_path/"rows.jsonl"), None)
+ for path in [out / "rl_agent_config.json", ckpt / "rl_agent_config.json"]:
+ cfg = json.loads(path.read_text())
+ assert cfg["temperature"] == [1.2, 1.2, 1.0]
+ assert "temperature_by_options" not in cfg
+ assert cfg["training"]["laya_train_calibration"]["score"]["items"] == 0
+ assert cfg["model_name"] == "laya-typed-decisions"
+ assert cfg["max_tokens_per_batch"] == 2048
+
+
+def test_no_duplicated_fitter_or_training_loop_remains():
+ single_text = (ROOT / "research" / "scripts" / "finetune_single_device.py").read_text(encoding="utf-8")
+ mps_text = (ROOT / "notebooks" / "laya_finetune_typed_decisions_mps.py").read_text(encoding="utf-8")
+ assert "fit_one_temp" not in single_text
+ assert "fit_temperature(" not in mps_text
+ assert "def preprocess" not in single_text
+ assert "while n_batches % args.grad_accum" not in mps_text