Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

feat: optimize DDP training hyperparameters and fix ECE calculation on typed-decisions

Nandakishor830d6245760b
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
index 6fc86180..b92c0ac6 100644
--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
@@ -225,6 +225,8 @@
" cfg = json.load(f)\n",
" cfg[\"gradient_checkpointing\"] = True\n",
" cfg[\"max_tokens_per_batch\"] = 4096\n",
+ " cfg[\"max_len\"] = 1024\n",
+ " cfg[\"head_max_len\"] = 256\n",
"\n",
" tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, \"tokenizer\"))\n",
" model = build_model(cfg, encoder_dir=os.path.join(model_dir, \"encoder\"))\n",
@@ -242,15 +244,15 @@
" all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n",
" my_items = all_items[rank::world_size]\n",
" \n",
- " # 4 Epochs with Cosine Annealing and soft-teacher guidance\n",
+ " # 4 Epochs with higher encoder rate & 1.0 soft-teacher guidance\n",
" EPOCHS = 4\n",
- " MICRO_BATCH = 8 # 8 sequences per GPU: fits comfortably up to 1024 tokens!\n",
+ " MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n",
" GRAD_ACCUM = 4 # Effective batch across 2 GPUs = 64 sequences (8 * 2 * 4)\n",
" GROUP_SIZE = 4 # GRPO baseline samples\n",
- " LR_ENCODER = 2.0e-5 # Encoder adaptation rate\n",
- " LR_HEAD = 1.2e-4 # Head adaptation rate\n",
- " SIGMA_START = 0.5 # Exploration noise\n",
- " SIGMA_END = 0.15\n",
+ " LR_ENCODER = 2.5e-5 # Encoder adaptation rate (higher for domain absorption)\n",
+ " LR_HEAD = 1.0e-4 # Controlled head rate to prevent logit explosion\n",
+ " SIGMA_START = 0.4 # Exploration noise\n",
+ " SIGMA_END = 0.1\n",
"\n",
" enc_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" in n]\n",
" head_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" not in n]\n",
@@ -260,8 +262,8 @@
" {\"params\": head_params, \"lr\": LR_HEAD}\n",
" ], weight_decay=0.01)\n",
" \n",
- " total_steps = (len(my_items) // (MICRO_BATCH * GRAD_ACCUM)) * EPOCHS\n",
- " scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_steps), eta_min=1e-6)\n",
+ " total_updates = (len(my_items) // (MICRO_BATCH * GRAD_ACCUM)) * EPOCHS\n",
+ " scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_updates), eta_min=1e-6)\n",
" scaler = torch.amp.GradScaler(\"cuda\", enabled=True)\n",
" \n",
" if rank == 0:\n",
@@ -311,11 +313,11 @@
" adv = r - r.mean(0, keepdim=True)\n",
" adv = adv / (adv.std() + 1e-6)\n",
" \n",
- " # 3. Policy gradient loss + auxiliary soft cross-entropy guidance (0.5 weight)\n",
+ " # 3. Policy gradient loss + full 1.0 soft cross-entropy guidance\n",
" logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)\n",
" loss_rl = -(adv * logp).mean()\n",
" loss_ce = -(target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()\n",
- " loss = (loss_rl + 0.5 * loss_ce) / GRAD_ACCUM + 0.0 * act.sum()\n",
+ " loss = (loss_rl + 1.0 * loss_ce) / GRAD_ACCUM + 0.0 * act.sum()\n",
" \n",
" scaler.scale(loss).backward()\n",
" accum_step += 1\n",
@@ -340,11 +342,11 @@
"\n",
" dist.barrier()\n",
" \n",
- " # Post-training temperature calibration on rank 0\n",
+ " # Post-training temperature calibration on rank 0 (balanced across all workflows)\n",
" if rank == 0:\n",
" print(\"\\nFitting post-training calibration temperatures...\")\n",
" model.eval()\n",
- " calib_items = all_items[:150]\n",
+ " calib_items = all_items[::15][:400]\n",
" calib_batch = collate_train_batch(calib_items, tok.pad_token_id)\n",
" with torch.no_grad():\n",
" with torch.autocast(\"cuda\", dtype=torch.float16):\n",
@@ -495,18 +497,18 @@
" if q_type == \"choice\":\n",
" keys = list(qdef[\"criteria\"].keys())\n",
" pred_choice = p_ans[\"choice\"]\n",
- " gold_label = g_ans[\"label\"]\n",
+ " gold_label = str(g_ans[\"label\"])\n",
" \n",
" is_corr = float(pred_choice == gold_label)\n",
" accuracies.append(is_corr)\n",
" all_corrects.append(is_corr)\n",
- " all_confs.append(p_ans[\"confidence\"])\n",
" \n",
" p_probs = np.array([p_ans[\"probabilities\"].get(k, 1e-6) for k in keys])\n",
" g_probs = np.array([g_ans[\"probabilities\"].get(k, 1e-6) for k in keys])\n",
" p_probs /= p_probs.sum()\n",
" g_probs /= g_probs.sum()\n",
" \n",
+ " all_confs.append(float(p_probs.max()))\n",
" soft_accuracies.append(float((p_probs * g_probs).sum()))\n",
" brier_scores.append(float(((p_probs - g_probs) ** 2).sum()))\n",
" tv_distances.append(float(0.5 * np.abs(p_probs - g_probs).sum()))\n",
@@ -522,7 +524,7 @@
" is_corr = float(pred_label == gold_label)\n",
" accuracies.append(is_corr)\n",
" all_corrects.append(is_corr)\n",
- " all_confs.append(max(p_val, 1.0 - p_val))\n",
+ " all_confs.append(float(max(p_val, 1.0 - p_val)))\n",
" \n",
" p_dist = np.array([1.0 - p_val, p_val])\n",
" g_dist = np.array([1.0 - g_val, g_val])\n",
@@ -539,12 +541,21 @@
" score_maes.append(abs(p_score - g_score))\n",
" within_one.append(float(abs(p_score - g_score) <= 1.0))\n",
" \n",
- " p_lvl = int(round(p_score))\n",
+ " # Use argmax of probabilities for discrete prediction\n",
+ " n_levels = len(qdef.get(\"criteria\", []))\n",
+ " p_probs_score = np.array([p_ans[\"probabilities\"].get(str(i), 0.0) for i in range(n_levels)])\n",
+ " if p_probs_score.sum() > 0:\n",
+ " p_probs_score /= p_probs_score.sum()\n",
+ " p_lvl = int(np.argmax(p_probs_score))\n",
+ " all_confs.append(float(p_probs_score.max()))\n",
+ " else:\n",
+ " p_lvl = int(round(p_score))\n",
+ " all_confs.append(0.5)\n",
+ " \n",
" g_lvl = int(g_ans.get(\"label\", int(round(g_score))))\n",
" is_corr = float(p_lvl == g_lvl)\n",
" accuracies.append(is_corr)\n",
" all_corrects.append(is_corr)\n",
- " all_confs.append(p_ans.get(\"confidence\", 0.5))\n",
"\n",
"laya_acc = np.mean(accuracies)\n",
"laya_soft_acc = np.mean(soft_accuracies)\n",