Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat(finetune): add single-device training script for any checkpoint

Wellington Fonseca369a8a537406
diff --git a/research/scripts/finetune_single_device.py b/research/scripts/finetune_single_device.py
new file mode 100644
index 00000000..0fd99d61
--- /dev/null
+++ b/research/scripts/finetune_single_device.py
@@ -0,0 +1,353 @@
+"""Fine-tune a Laya checkpoint on a single device (CPU or GPU).
+
+Reproduces the 2xT4 notebook's training loop without DDP, so any checkpoint
+(including the multilingual one) can be fine-tuned on one GPU or on a CPU.
+Preprocesses a JSONL dataset into tokenized items, trains with RLCD, fits one
+calibration temperature per question type, and saves a loadable checkpoint.
+
+Each JSONL line is one case: {"state": "...", "questions": {...}, "gold": {...}}
+where questions use the choice / score / noul primitives and gold carries the
+teacher probabilities plus a label, matching the schema in docs/finetune.md.
+
+Usage:
+ python research/scripts/finetune_single_device.py \
+ --data dataset.jsonl --model-dir /path/to/multilingual --output-dir out/
+"""
+
+import argparse
+import json
+import os
+import random
+import sys
+
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.insert(0, ROOT)
+
+import torch
+from huggingface_hub import snapshot_download
+from safetensors.torch import load_file, save_file
+from transformers import AutoTokenizer
+
+from laya.agent import _fix_tokenizer_config
+from laya.common import (
+ QTYPES,
+ build_model,
+ build_sequence,
+ proper_reward,
+ render_options,
+)
+
+
+def build_training_item(tok, cfg, state, q, gold_q):
+ t = q["type"]
+ crit = q.get("criteria", {})
+ if t == "choice":
+ keys = list(crit.keys())
+ target = [gold_q["probabilities"].get(k, 0.0) for k in keys]
+ elif t == "noul":
+ target = [
+ gold_q["probabilities"].get("false", 0.5),
+ gold_q["probabilities"].get("true", 0.5),
+ ]
+ else:
+ n_levels = len(crit) if isinstance(crit, list) else 4
+ target = [gold_q["probabilities"].get(str(i), 0.0) for i in range(n_levels)]
+ total = sum(target)
+ target = (
+ [v / total for v in target] if total > 0 else [1.0 / len(target)] * len(target)
+ )
+ seq, markers = build_sequence(
+ tok,
+ state,
+ {"t": t, "ins": q["instructions"], "crit": crit},
+ cfg["max_len"],
+ cfg["head_max_len"],
+ )
+ if len(markers) != len(render_options({"t": t, "crit": crit})):
+ return None
+ return {
+ "ids": seq,
+ "markers": markers,
+ "qtype": QTYPES[t],
+ "target": target,
+ "label": target.index(max(target)),
+ }
+
+
+def preprocess(tok, cfg, data_path):
+ items = []
+ with open(data_path, encoding="utf-8") as f:
+ for line in f:
+ row = json.loads(line)
+ for qid, q in row["questions"].items():
+ if qid not in row["gold"]:
+ continue
+ item = build_training_item(tok, cfg, row["state"], q, row["gold"][qid])
+ if item is not None:
+ items.append(item)
+ return items
+
+
+def collate(items, pad_id):
+ n, length = len(items), max(len(it["ids"]) for it in items)
+ kmax = max(len(it["markers"]) for it in items)
+ ids = torch.full((n, length), pad_id, dtype=torch.long)
+ att = torch.zeros((n, length), dtype=torch.long)
+ mpos = torch.zeros((n, kmax), dtype=torch.long)
+ mmask = torch.zeros((n, kmax), dtype=torch.bool)
+ target = torch.zeros((n, kmax), dtype=torch.float32)
+ for i, it in enumerate(items):
+ ids[i, : len(it["ids"])] = torch.tensor(it["ids"])
+ att[i, : len(it["ids"])] = 1
+ k = len(it["markers"])
+ mpos[i, :k] = torch.tensor(it["markers"])
+ mmask[i, :k] = True
+ target[i, : len(it["target"])] = torch.tensor(it["target"], dtype=torch.float32)
+ return {
+ "input_ids": ids,
+ "attention_mask": att,
+ "marker_pos": mpos,
+ "marker_mask": mmask,
+ "target": target,
+ "qtype": torch.tensor([it["qtype"] for it in items]),
+ }
+
+
+def forward(model, batch, device, use_amp):
+ if use_amp:
+ with torch.autocast("cuda", dtype=torch.float16):
+ return model(
+ batch["input_ids"].to(device),
+ batch["attention_mask"].to(device),
+ batch["marker_pos"].to(device),
+ batch["marker_mask"].to(device),
+ batch["qtype"].to(device),
+ )
+ return model(
+ batch["input_ids"].to(device),
+ batch["attention_mask"].to(device),
+ batch["marker_pos"].to(device),
+ batch["marker_mask"].to(device),
+ batch["qtype"].to(device),
+ )
+
+
+def fit_one_temp(sel):
+ if len(sel) < 10:
+ return 1.0
+ kmax = max(len(z) for z, _ in sel)
+ Z = torch.full((len(sel), kmax), -1e4)
+ T = torch.zeros((len(sel), kmax))
+ for i, (z, t) in enumerate(sel):
+ Z[i, : len(z)] = torch.tensor(z)
+ T[i, : len(t)] = torch.tensor(t, dtype=torch.float32)
+ log_t = torch.zeros(1, requires_grad=True)
+ opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)
+
+ def closure():
+ opt.zero_grad()
+ loss = -(T * torch.log_softmax(Z / log_t.exp(), -1)).sum(-1).mean()
+ loss.backward()
+ return loss
+
+ opt.step(closure)
+ return float(torch.clamp(log_t.exp(), 0.1, 10.0).item())
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Fine-tune a Laya checkpoint on one device."
+ )
+ parser.add_argument(
+ "--data", required=True, help="JSONL dataset of {state, questions, gold} cases."
+ )
+ parser.add_argument(
+ "--model-dir",
+ default=None,
+ help="Checkpoint directory; defaults to the multilingual subfolder.",
+ )
+ parser.add_argument("--output-dir", default="laya_finetuned")
+ parser.add_argument(
+ "--device", default="auto", choices=["auto", "cpu", "cuda", "mps"]
+ )
+ parser.add_argument("--epochs", type=int, default=4)
+ parser.add_argument("--seed", type=int, default=0)
+ args = parser.parse_args()
+
+ if args.device == "auto":
+ device = (
+ torch.device("cuda", 0)
+ if torch.cuda.is_available()
+ else torch.device("cpu")
+ )
+ else:
+ device = torch.device(args.device)
+ use_amp = device.type == "cuda"
+ torch.manual_seed(args.seed)
+ random.seed(args.seed)
+ torch.set_num_threads(int(os.environ.get("OMP_NUM_THREADS", os.cpu_count() or 1)))
+ print("Device:", device)
+
+ if args.model_dir:
+ model_dir = args.model_dir
+ else:
+ model_dir = os.path.join(
+ snapshot_download(
+ "convaiinnovations/laya",
+ allow_patterns=["multilingual/*", "multilingual/*/*"],
+ ),
+ "multilingual",
+ )
+ _fix_tokenizer_config(model_dir)
+
+ with open(os.path.join(model_dir, "rl_agent_config.json")) as f:
+ cfg = json.load(f)
+ cfg["gradient_checkpointing"] = use_amp
+ cfg["max_tokens_per_batch"] = 4096
+ cfg["max_len"] = 1024
+ cfg["head_max_len"] = 256
+
+ tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, "tokenizer"))
+ model = build_model(cfg, encoder_dir=os.path.join(model_dir, "encoder"))
+ model.load_state_dict(
+ load_file(os.path.join(model_dir, "model.safetensors")), strict=True
+ )
+ if use_amp:
+ model.encoder.gradient_checkpointing_enable(
+ gradient_checkpointing_kwargs={"use_reentrant": False}
+ )
+ model.head_checkpointing = True
+ model.to(device)
+ model.train()
+
+ all_items = preprocess(tok, cfg, args.data)
+ if not all_items:
+ raise SystemExit("No training items produced from --data.")
+
+ order = list(range(len(all_items)))
+ random.shuffle(order)
+ n_calib = min(400, len(all_items) // 10)
+ calib_items = [all_items[i] for i in sorted(order[:n_calib])]
+ train_items = [all_items[i] for i in sorted(order[n_calib:])]
+
+ micro_batch = 8
+ group_size = 4
+ enc_params = [p for n, p in model.named_parameters() if "encoder." in n]
+ head_params = [p for n, p in model.named_parameters() if "encoder." not in n]
+ optimizer = torch.optim.AdamW(
+ [{"params": enc_params, "lr": 2.5e-5}, {"params": head_params, "lr": 1.0e-4}],
+ weight_decay=0.01,
+ )
+ steps_per_epoch = max(1, (len(train_items) + micro_batch - 1) // micro_batch)
+ scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
+ optimizer, T_max=steps_per_epoch * args.epochs, eta_min=1e-6
+ )
+ scaler = torch.amp.GradScaler("cuda", enabled=True) if use_amp else None
+
+ print(
+ "Train items: {} ({} held out) | {} epochs | {} steps/epoch".format(
+ len(train_items), len(calib_items), args.epochs, steps_per_epoch
+ )
+ )
+
+ for epoch in range(args.epochs):
+ random.seed(args.seed + epoch)
+ random.shuffle(train_items)
+ sigma = 0.4 + (0.1 - 0.4) * (epoch / max(1, args.epochs - 1))
+ epoch_loss, n_batches = 0.0, 0
+ optimizer.zero_grad(set_to_none=True)
+ for b_idx in range(0, len(train_items), micro_batch):
+ chunk = train_items[b_idx : b_idx + micro_batch]
+ if not chunk:
+ continue
+ batch = collate(chunk, tok.pad_token_id)
+ logits, _act = forward(model, batch, device, use_amp)
+ logits = logits.float()
+ mask = batch["marker_mask"].to(device)
+ k = mask.sum(-1, keepdim=True).float()
+ target = batch["target"].to(device)
+
+ eps = (
+ torch.randn((group_size,) + logits.shape, device=device) * sigma * mask
+ )
+ eps = (eps - eps.sum(-1, keepdim=True) / k) * mask
+ z = logits.detach().unsqueeze(0) + eps
+ q = torch.softmax(z.masked_fill(~mask, -1e4), -1)
+
+ with torch.no_grad():
+ r = proper_reward(
+ q,
+ target.unsqueeze(0),
+ batch["qtype"].to(device),
+ mask,
+ w_sph=0.75,
+ w_rps=1.0,
+ )
+ adv = r - r.mean(0, keepdim=True)
+ adv = adv / (adv.std() + 1e-6)
+
+ logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma**2)
+ loss = (
+ -(adv * logp).mean()
+ - (target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1))
+ .sum(-1)
+ .mean()
+ )
+
+ if scaler:
+ scaler.scale(loss).backward()
+ scaler.unscale_(optimizer)
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ scaler.step(optimizer)
+ scaler.update()
+ else:
+ loss.backward()
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+ optimizer.step()
+ scheduler.step()
+ optimizer.zero_grad(set_to_none=True)
+ epoch_loss += loss.item()
+ n_batches += 1
+ print(
+ "Epoch {}/{} | avg loss {:.4f}".format(
+ epoch + 1, args.epochs, epoch_loss / max(1, n_batches)
+ ),
+ flush=True,
+ )
+
+ model.eval()
+ calib_preds = []
+ with torch.no_grad():
+ for c_idx in range(0, len(calib_items), 16):
+ c_chunk = calib_items[c_idx : c_idx + 16]
+ cb = collate(c_chunk, tok.pad_token_id)
+ l_sub, _ = forward(model, cb, device, use_amp)
+ l_np = l_sub.float().cpu().numpy()
+ for r, it in enumerate(c_chunk):
+ kk = len(it["markers"])
+ calib_preds.append((it["qtype"], l_np[r, :kk], it["target"]))
+
+ fitted = [1.2, 1.2, 1.2]
+ for qt in range(3):
+ sel = [(z, t) for q_type, z, t in calib_preds if q_type == qt]
+ if sel:
+ fitted[qt] = fit_one_temp(sel)
+ print("Fitted temperatures (choice, score, noul):", [round(t, 3) for t in fitted])
+
+ os.makedirs(args.output_dir, exist_ok=True)
+ save_file(
+ {key: val.half().contiguous().cpu() for key, val in model.state_dict().items()},
+ os.path.join(args.output_dir, "model.safetensors"),
+ )
+ model.encoder.config.save_pretrained(os.path.join(args.output_dir, "encoder"))
+ tok.save_pretrained(os.path.join(args.output_dir, "tokenizer"))
+ cfg["fine_tuned"] = True
+ cfg["temperature"] = fitted
+ cfg.pop("temperature_by_options", None)
+ with open(os.path.join(args.output_dir, "rl_agent_config.json"), "w") as f:
+ json.dump(cfg, f, indent=2)
+ print("Saved checkpoint to", args.output_dir)
+
+
+if __name__ == "__main__":
+ main()