Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
refactor(notebook): converge Kaggle 2xT4 DDP script onto canonical training primitives
Bruce-Yii13bcc03302a0
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynbindex 800e7edf..58fc9e44 100644--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb@@ -4,7 +4,7 @@"cell_type": "markdown","metadata": {},"source": [- "# Fine-Tuning Laya on `LocalLLaMA/typed-decisions` (Kaggle GPU T4 \u00d72 DDP)\n",+ "# Fine-Tuning Laya on `LocalLLaMA/typed-decisions` (Kaggle GPU T4 ×2 DDP)\n","\n","[](https://pypi.org/project/laya/)\n","[](https://huggingface.co/convaiinnovations/laya)\n",@@ -14,7 +14,7 @@"\n","---\n","\n",- "### \u26a0\ufe0f Kaggle Notebook Settings:\n",+ "### ⚠️ Kaggle Notebook Settings:\n","In the right-hand sidebar panel under **Notebook options**:\n","* **Accelerator:** Select **`GPU T4 x2`** (ensure both GPUs are allocated)\n","* **Internet:** Set to **`On`**\n",@@ -91,7 +91,7 @@"from transformers import AutoTokenizer\n","from huggingface_hub import snapshot_download\n","from laya.agent import _fix_tokenizer_config\n",- "from laya.common import build_sequence, render_options, QTYPES\n",+ "from laya.train import items_from_rows\n","\n","MODEL_ID = \"convaiinnovations/laya\"\n","print(f\"Fetching tokenizer and config from {MODEL_ID}...\")\n",@@ -105,46 +105,16 @@"print(\"Downloading LocalLLaMA/typed-decisions (train split)...\")\n","ds_train = load_dataset(\"LocalLLaMA/typed-decisions\", \"all\", split=\"train\")\n","\n",- "def build_training_item(state, q, gold_q):\n",- " t = q[\"type\"]\n",- " crit = q.get(\"criteria\", {})\n",- " if t == \"choice\":\n",- " keys = list(crit.keys())\n",- " target = [gold_q[\"probabilities\"].get(k, 0.0) for k in keys]\n",- " elif t == \"noul\":\n",- " target = [gold_q[\"probabilities\"].get(\"false\", 0.5), gold_q[\"probabilities\"].get(\"true\", 0.5)]\n",- " elif t == \"score\":\n",- " n_levels = len(crit) if isinstance(crit, list) else 4\n",- " target = [gold_q[\"probabilities\"].get(str(i), 0.0) for i in range(n_levels)]\n",- " \n",- " s = sum(target)\n",- " target = [v / s for v in target] if s > 0 else [1.0 / len(target)] * len(target)\n",- " label = target.index(max(target))\n",- " k = len(render_options({\"t\": t, \"crit\": crit}))\n",- " \n",- " seq, markers = build_sequence(tok, state, {\"t\": t, \"ins\": q[\"instructions\"], \"crit\": crit}, cfg[\"max_len\"], cfg[\"head_max_len\"])\n",- " if len(markers) != k:\n",- " return None\n",- " return {\n",- " \"ids\": seq,\n",- " \"markers\": markers,\n",- " \"qtype\": QTYPES[t],\n",- " \"target\": target,\n",- " \"label\": label\n",- " }\n",- "\n",- "items = []\n",+ "rows = []\n","for row in ds_train:\n",- " state = json.loads(row[\"state\"])\n",- " questions = json.loads(row[\"questions\"])\n",- " gold = json.loads(row[\"gold\"])\n",- " for qid, q in questions.items():\n",- " if qid in gold:\n",- " it = build_training_item(state, q, gold[qid])\n",- " if it:\n",- " items.append(it)\n",+ " rows.append({\n",+ " \"state\": json.loads(row[\"state\"]),\n",+ " \"questions\": json.loads(row[\"questions\"]),\n",+ " \"gold\": json.loads(row[\"gold\"]),\n",+ " })\n","\n",- "print(f\"Preprocessed {len(items)} training sequences across 1,200 cases.\")\n",+ "items, skipped = items_from_rows(tok, rows, cfg[\"max_len\"], cfg[\"head_max_len\"])\n",+ "print(f\"Preprocessed {len(items)} training items (skipped {skipped}) across 1,200 cases.\")\n","torch.save(items, \"/kaggle/working/train_items.pt\")\n","print(\"Saved preprocessed items to /kaggle/working/train_items.pt\")\n"],@@ -164,58 +134,33 @@"metadata": {},"source": ["%%writefile /kaggle/working/train_ddp.py\n",+ "\"\"\"2xT4 DDP training script for the typed-decisions fine-tune.\n",+ "\n",+ "Training semantics -- item construction, encoding/collation, the RLCD objective,\n",+ "the sigma schedule, the calibration holdout and the temperature fit -- come from\n",+ "the canonical primitives in `laya.train` / `laya.common` / `laya.calibrate`.\n",+ "What intentionally stays notebook-private is the distributed orchestration:\n",+ "process-group setup, rank/local-rank wiring, the DDP wrapper, equal-rank\n",+ "sharding and the effective-batch accounting.\n",+ "\"\"\"\n","import os, sys, time, json, random, math\n","import numpy as np\n","import torch\n","import torch.distributed as dist\n","from torch.nn.parallel import DistributedDataParallel as DDP\n",- "from safetensors.torch import load_file, save_file\n",+ "from safetensors.torch import load_file\n","from transformers import AutoTokenizer\n",- "from laya.common import build_model, proper_reward, QTYPES, TEMP_MIN, TEMP_MAX\n",- "\n",- "def collate_train_batch(items, pad_id):\n",- " n, L = len(items), max(len(it[\"ids\"]) for it in items)\n",- " kmax = max(len(it[\"markers\"]) for it in items)\n",- " ids = torch.full((n, L), pad_id, dtype=torch.long)\n",- " att = torch.zeros((n, L), dtype=torch.long)\n",- " mpos = torch.zeros((n, kmax), dtype=torch.long)\n",- " mmask = torch.zeros((n, kmax), dtype=torch.bool)\n",- " target = torch.zeros((n, kmax), dtype=torch.float32)\n",- " for i, it in enumerate(items):\n",- " ids[i, : len(it[\"ids\"])] = torch.tensor(it[\"ids\"])\n",- " att[i, : len(it[\"ids\"])] = 1\n",- " k = len(it[\"markers\"])\n",- " mpos[i, :k] = torch.tensor(it[\"markers\"])\n",- " mmask[i, :k] = True\n",- " target[i, : len(it[\"target\"])] = torch.tensor(it[\"target\"], dtype=torch.float32)\n",- " return {\n",- " \"input_ids\": ids,\n",- " \"attention_mask\": att,\n",- " \"marker_pos\": mpos,\n",- " \"marker_mask\": mmask,\n",- " \"target\": target,\n",- " \"qtype\": torch.tensor([it[\"qtype\"] for it in items]),\n",- " \"label\": torch.tensor([it[\"label\"] for it in items])\n",- " }\n","\n",- "def fit_one_temp(sel):\n",- " if len(sel) < 10:\n",- " return 1.0\n",- " kmax = max(len(z) for z, _ in sel)\n",- " Z = torch.full((len(sel), kmax), -1e4)\n",- " T = torch.zeros((len(sel), kmax))\n",- " for i, (z, t) in enumerate(sel):\n",- " Z[i, :len(z)] = torch.tensor(z)\n",- " T[i, :len(t)] = torch.tensor(t, dtype=torch.float32)\n",- " log_t = torch.zeros(1, requires_grad=True)\n",- " opt = torch.optim.LBFGS([log_t], lr=0.1, max_iter=100)\n",- " def closure():\n",- " opt.zero_grad()\n",- " loss = -(T * torch.log_softmax(Z / log_t.exp(), -1)).sum(-1).mean()\n",- " loss.backward()\n",- " return loss\n",- " opt.step(closure)\n",- " return float(torch.clamp(log_t.exp(), TEMP_MIN, TEMP_MAX).item())\n",+ "from laya.common import build_model, collate_items\n",+ "from laya.calibrate import fit_temperature_map\n",+ "from laya.train import (\n",+ " calibration_records,\n",+ " encode_item,\n",+ " rlcd_loss,\n",+ " save_checkpoint,\n",+ " sigma_at,\n",+ " split_calibration,\n",+ ")\n","\n","def main():\n"," dist.init_process_group(\"nccl\")\n",@@ -227,46 +172,46 @@"\n"," model_dir = sys.argv[1]\n"," output_dir = sys.argv[2]\n",- " \n",+ "\n"," with open(os.path.join(model_dir, \"rl_agent_config.json\")) as f:\n"," cfg = json.load(f)\n"," cfg[\"gradient_checkpointing\"] = True\n"," cfg[\"max_tokens_per_batch\"] = 4096\n"," cfg[\"max_len\"] = 1024\n"," cfg[\"head_max_len\"] = 256\n",+ " max_len = cfg[\"max_len\"]\n",+ " head_max_len = cfg[\"head_max_len\"]\n","\n"," tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, \"tokenizer\"))\n"," model = build_model(cfg, encoder_dir=os.path.join(model_dir, \"encoder\"))\n",- " \n",+ "\n"," weights = load_file(os.path.join(model_dir, \"model.safetensors\"))\n"," model.load_state_dict(weights, strict=True)\n",- " \n",+ "\n"," model.encoder.gradient_checkpointing_enable(gradient_checkpointing_kwargs={\"use_reentrant\": False})\n"," model.head_checkpointing = True\n"," model.to(device)\n"," model.train()\n","\n"," ddp_model = DDP(model, device_ids=[local_rank], find_unused_parameters=True)\n",- " \n",+ "\n"," all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n","\n",- " # Hold the calibration slice out of training before sharding. Temperatures fitted on\n",- " # items the run has already trained on measure the fit rather than the calibration: the\n",- " # model is near-certain and near-correct on them, so the optimiser has nothing to soften\n",- " # and returns a degenerate scale. The seed is fixed and rank-independent, so every rank\n",- " # withholds exactly the same items and none of them reaches a training batch.\n",- " CALIB_MAX = 400\n",- " order = list(range(len(all_items)))\n",- " random.Random(20260922).shuffle(order)\n",- " n_calib = min(CALIB_MAX, len(all_items) // 10)\n",- " calib_items = [all_items[i] for i in sorted(order[:n_calib])]\n",- " train_items = [all_items[i] for i in sorted(order[n_calib:])]\n",- " # Trim before sharding so every rank gets the same count (drops at most\n # world_size-1 items). Otherwise ranks can run different numbers of\n # micro-batches and DDP deadlocks on the unmatched all-reduce.\n # See https://github.com/NandhaKishorM/laya/issues/678\n train_items = train_items[:len(train_items) // world_size * world_size]\n my_items = train_items[rank::world_size]\n",- " \n",+ " # Held out before sharding: `split_calibration` is seed-deterministic and\n",+ " # rank-independent, so every rank withholds the same items, and the shard\n",+ " # split happens after -- the calibration slice never reaches a training batch.\n",+ " train_items, calib_items = split_calibration(all_items, calib_max=400, calib_frac=0.1, seed=20260922)\n",+ "\n",+ " # Trim before sharding so every rank gets the same count (drops at most\n",+ " # world_size-1 items). Otherwise ranks can run different numbers of\n",+ " # micro-batches and DDP deadlocks on the unmatched all-reduce (#678).\n",+ " train_items = train_items[:len(train_items) // world_size * world_size]\n",+ " my_items = train_items[rank::world_size]\n",+ "\n"," EPOCHS = 4\n"," MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n"," GRAD_ACCUM = 4 # Effective batch across 2 GPUs = 64 sequences (8 * 2 * 4)\n",- " GROUP_SIZE = 4 # GRPO baseline samples\n",+ " GROUP_SIZE = 4 # RLCD baseline samples\n"," LR_ENCODER = 2.5e-5 # Encoder adaptation rate\n"," LR_HEAD = 1.0e-4 # Head adaptation rate\n"," SIGMA_START = 0.4 # Exploration noise\n",@@ -274,37 +219,38 @@"\n"," enc_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" in n]\n"," head_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" not in n]\n",- " \n",+ "\n"," optimizer = torch.optim.AdamW([\n"," {\"params\": enc_params, \"lr\": LR_ENCODER},\n"," {\"params\": head_params, \"lr\": LR_HEAD}\n"," ], weight_decay=0.01)\n",- " \n",+ "\n"," total_updates = (len(my_items) // (MICRO_BATCH * GRAD_ACCUM)) * EPOCHS\n"," scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_updates), eta_min=1e-6)\n"," scaler = torch.amp.GradScaler(\"cuda\", enabled=True)\n",- " \n",+ "\n"," if rank == 0:\n"," print(f\"Starting 2xT4 DDP training: {len(train_items)} train items ({len(calib_items)} held out for calibration) | {len(my_items)} per rank | {EPOCHS} epochs\")\n"," t0 = time.time()\n",- " \n",+ "\n"," for epoch in range(EPOCHS):\n"," random.seed(42 + epoch + rank)\n"," random.shuffle(my_items)\n"," epoch_loss, n_batches = 0.0, 0\n"," optimizer.zero_grad(set_to_none=True)\n"," accum_step = 0\n",- " \n",- " progress = epoch / max(1, EPOCHS - 1)\n",- " sigma = SIGMA_START + (SIGMA_END - SIGMA_START) * progress\n",- " \n",+ "\n",+ " sigma = sigma_at(epoch, EPOCHS, SIGMA_START, SIGMA_END)\n",+ " steps_per_epoch = math.ceil(len(my_items) / MICRO_BATCH)\n",+ "\n"," for b_idx in range(0, len(my_items), MICRO_BATCH):\n"," chunk = my_items[b_idx:b_idx + MICRO_BATCH]\n"," if not chunk:\n"," continue\n",- " \n",- " batch = collate_train_batch(chunk, tok.pad_token_id)\n",- " \n",+ "\n",+ " encoded = [encode_item(tok, it, max_len, head_max_len) for it in chunk]\n",+ " batch = collate_items([encoded], tok.pad_token_id)\n",+ "\n"," with torch.autocast(\"cuda\", dtype=torch.float16):\n"," logits, act = ddp_model(\n"," batch[\"input_ids\"].to(device),\n",@@ -313,33 +259,22 @@" batch[\"marker_mask\"].to(device),\n"," batch[\"qtype\"].to(device)\n"," )\n",- " \n",+ "\n"," logits = logits.float()\n"," mask = batch[\"marker_mask\"].to(device)\n",- " k = mask.sum(-1, keepdim=True).float()\n"," target = batch[\"target\"].to(device)\n",- " \n",- " # 1. Sample G noisy logit distributions with zero-mean projection\n",- " eps = torch.randn((GROUP_SIZE,) + logits.shape, device=device) * sigma * mask\n",- " eps = (eps - eps.sum(-1, keepdim=True) / k) * mask\n",- " z = logits.detach().unsqueeze(0) + eps\n",- " q = torch.softmax(z.masked_fill(~mask, -1e4), -1)\n",- " \n",- " # 2. Evaluate proper scoring reward (w_sph=0.75 for soft target matching)\n",- " with torch.no_grad():\n",- " r = proper_reward(q, target.unsqueeze(0), batch[\"qtype\"].to(device), mask, w_sph=0.75, w_rps=1.0)\n",- " adv = r - r.mean(0, keepdim=True)\n",- " adv = adv / (adv.std() + 1e-6)\n",- " \n",- " # 3. Policy gradient loss + full 1.0 soft cross-entropy guidance\n",- " logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)\n",- " loss_rl = -(adv * logp).mean()\n",- " loss_ce = -(target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()\n",- " loss = (loss_rl + 1.0 * loss_ce) / GRAD_ACCUM + 0.0 * act.sum()\n",- " \n",- " scaler.scale(loss).backward()\n",+ "\n",+ " # Canonical objective: RLCD policy gradient over noisy logits + full\n",+ " # soft cross-entropy guidance (`laya.train.rlcd_loss`).\n",+ " loss = rlcd_loss(logits, target, mask, batch[\"qtype\"].to(device), sigma,\n",+ " samples=GROUP_SIZE, w_sph=0.75, w_rps=1.0)\n",+ "\n",+ " window_start = (accum_step // GRAD_ACCUM) * GRAD_ACCUM\n",+ " window_size = min(GRAD_ACCUM, steps_per_epoch - window_start)\n",+ " scaled = loss / window_size\n",+ " scaler.scale(scaled).backward()\n"," accum_step += 1\n",- " \n",+ "\n"," if accum_step % GRAD_ACCUM == 0 or (b_idx + MICRO_BATCH) >= len(my_items):\n"," scaler.unscale_(optimizer)\n"," torch.nn.utils.clip_grad_norm_(ddp_model.parameters(), 1.0)\n",@@ -347,13 +282,13 @@" scaler.update()\n"," scheduler.step()\n"," optimizer.zero_grad(set_to_none=True)\n",- " \n",- " epoch_loss += loss.item() * GRAD_ACCUM\n",+ "\n",+ " epoch_loss += loss.item()\n"," n_batches += 1\n",- " \n",+ "\n"," if rank == 0 and (n_batches % 50) == 0:\n"," cur_lr = scheduler.get_last_lr()[0]\n",- " print(f\" Epoch {epoch+1}/{EPOCHS} | Step {n_batches} | Loss: {loss.item()*GRAD_ACCUM:.4f} | Reward: {r.mean().item():.3f} | LR: {cur_lr:.2e}\")\n",+ " print(f\" Epoch {epoch+1}/{EPOCHS} | Step {n_batches} | Loss: {loss.item():.4f} | LR: {cur_lr:.2e}\")\n","\n"," if rank == 0:\n"," print(f\"=== Epoch {epoch+1}/{EPOCHS} Completed in {time.time()-t0:.1f}s | Avg Loss: {epoch_loss/max(1, n_batches):.4f} ===\")\n",@@ -364,11 +299,7 @@" # OOM, or Kaggle session timeout doesn't lose all prior training.\n"," if rank == 0:\n"," ckpt_dir = os.path.join(output_dir, \"checkpoint_latest\")\n",- " os.makedirs(ckpt_dir, exist_ok=True)\n",- " ckpt_sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}\n",- " save_file(ckpt_sd, os.path.join(ckpt_dir, \"model.safetensors\"))\n",- " model.encoder.config.save_pretrained(os.path.join(ckpt_dir, \"encoder\"))\n",- " tok.save_pretrained(os.path.join(ckpt_dir, \"tokenizer\"))\n",+ " save_checkpoint(model, tok, dict(cfg, max_len=max_len, head_max_len=head_max_len), ckpt_dir)\n"," with open(os.path.join(ckpt_dir, \"checkpoint_meta.json\"), \"w\") as f:\n"," json.dump({\n"," \"epoch\": epoch + 1,\n",@@ -378,54 +309,32 @@" print(f\" Saved rolling checkpoint (epoch {epoch+1}/{EPOCHS}) to {ckpt_dir}\")\n","\n"," dist.barrier()\n",- " \n",- " # Post-training temperature calibration on rank 0 (micro-batched in chunks of 16 to prevent OOM)\n",+ "\n",+ " # Post-training temperature calibration on rank 0\n"," if rank == 0:\n"," print(\"\\nFitting post-training calibration temperatures...\")\n"," del optimizer, scaler, scheduler\n"," torch.cuda.empty_cache()\n"," model.eval()\n",- " # calib_items was held out above and never entered a training batch\n",- " calib_preds = []\n",- " with torch.no_grad():\n",- " for c_idx in range(0, len(calib_items), 16):\n",- " c_chunk = calib_items[c_idx:c_idx + 16]\n",- " cb = collate_train_batch(c_chunk, tok.pad_token_id)\n",- " with torch.autocast(\"cuda\", dtype=torch.float16):\n",- " l_sub, _ = model(\n",- " cb[\"input_ids\"].to(device),\n",- " cb[\"attention_mask\"].to(device),\n",- " cb[\"marker_pos\"].to(device),\n",- " cb[\"marker_mask\"].to(device),\n",- " cb[\"qtype\"].to(device)\n",- " )\n",- " l_np = l_sub.float().cpu().numpy()\n",- " for r, it in enumerate(c_chunk):\n",- " k = len(it[\"markers\"])\n",- " calib_preds.append((it[\"qtype\"], l_np[r, :k], it[\"target\"]))\n",- " \n",- " fitted_temps = [1.2, 1.2, 1.2]\n"," try:\n",- " for qt in range(3):\n",- " sel = [(z, t) for q_type, z, t in calib_preds if q_type == qt]\n",- " if sel:\n",- " fitted_temps[qt] = fit_one_temp(sel)\n",+ " records = calibration_records(model, tok, calib_items, device, max_len, head_max_len, batch_size=16)\n",+ " fitted = fit_temperature_map(records)\n",+ " fitted_temps = fitted[\"temperature\"]\n"," print(\"Fitted calibration temperatures (choice, score, noul):\", [round(t, 3) for t in fitted_temps])\n"," except Exception as e:\n"," print(\"Temperature fitting fallback:\", e)\n",- " os.makedirs(output_dir, exist_ok=True)\n",- " sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}\n",- " save_file(sd, os.path.join(output_dir, \"model.safetensors\"))\n",- " model.encoder.config.save_pretrained(os.path.join(output_dir, \"encoder\"))\n",- " tok.save_pretrained(os.path.join(output_dir, \"tokenizer\"))\n",- " \n",- " cfg[\"fine_tuned\"] = True\n",- " cfg[\"model_name\"] = \"laya-typed-decisions\"\n",- " cfg[\"temperature\"] = fitted_temps\n",- " # This fit is per type; inherited bucket overrides would hide the new values.\n",- " cfg.pop(\"temperature_by_options\", None)\n",- " with open(os.path.join(output_dir, \"rl_agent_config.json\"), \"w\") as f:\n",- " json.dump(cfg, f, indent=2)\n",+ " fitted_temps, fitted = None, {\"temperature_by_options\": {}}\n",+ "\n",+ " out_cfg = dict(cfg, max_len=max_len, head_max_len=head_max_len)\n",+ " out_cfg[\"fine_tuned\"] = True\n",+ " out_cfg[\"model_name\"] = \"laya-typed-decisions\"\n",+ " if fitted_temps is not None:\n",+ " out_cfg[\"temperature\"] = fitted_temps\n",+ " # This fit carries per-bucket overrides; inherited ones would shadow them.\n",+ " out_cfg.pop(\"temperature_by_options\", None)\n",+ " if fitted[\"temperature_by_options\"]:\n",+ " out_cfg[\"temperature_by_options\"] = fitted[\"temperature_by_options\"]\n",+ " save_checkpoint(model, tok, out_cfg, output_dir)\n"," print(f\"Model successfully saved to {output_dir}!\")\n","\n"," dist.destroy_process_group()\n",@@ -913,4 +822,4 @@},"nbformat": 4,"nbformat_minor": 4-}+}\ No newline at end of file