Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
feat: optimize DDP training hyperparameters and fix ECE calculation on typed-decisions
Nandakishor830d6245760b
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynbindex 6fc86180..b92c0ac6 100644--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb@@ -225,6 +225,8 @@" cfg = json.load(f)\n"," cfg[\"gradient_checkpointing\"] = True\n"," cfg[\"max_tokens_per_batch\"] = 4096\n",+ " cfg[\"max_len\"] = 1024\n",+ " cfg[\"head_max_len\"] = 256\n","\n"," tok = AutoTokenizer.from_pretrained(os.path.join(model_dir, \"tokenizer\"))\n"," model = build_model(cfg, encoder_dir=os.path.join(model_dir, \"encoder\"))\n",@@ -242,15 +244,15 @@" all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n"," my_items = all_items[rank::world_size]\n"," \n",- " # 4 Epochs with Cosine Annealing and soft-teacher guidance\n",+ " # 4 Epochs with higher encoder rate & 1.0 soft-teacher guidance\n"," EPOCHS = 4\n",- " MICRO_BATCH = 8 # 8 sequences per GPU: fits comfortably up to 1024 tokens!\n",+ " MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n"," GRAD_ACCUM = 4 # Effective batch across 2 GPUs = 64 sequences (8 * 2 * 4)\n"," GROUP_SIZE = 4 # GRPO baseline samples\n",- " LR_ENCODER = 2.0e-5 # Encoder adaptation rate\n",- " LR_HEAD = 1.2e-4 # Head adaptation rate\n",- " SIGMA_START = 0.5 # Exploration noise\n",- " SIGMA_END = 0.15\n",+ " LR_ENCODER = 2.5e-5 # Encoder adaptation rate (higher for domain absorption)\n",+ " LR_HEAD = 1.0e-4 # Controlled head rate to prevent logit explosion\n",+ " SIGMA_START = 0.4 # Exploration noise\n",+ " SIGMA_END = 0.1\n","\n"," enc_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" in n]\n"," head_params = [p for n, p in ddp_model.named_parameters() if \"encoder.\" not in n]\n",@@ -260,8 +262,8 @@" {\"params\": head_params, \"lr\": LR_HEAD}\n"," ], weight_decay=0.01)\n"," \n",- " total_steps = (len(my_items) // (MICRO_BATCH * GRAD_ACCUM)) * EPOCHS\n",- " scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_steps), eta_min=1e-6)\n",+ " total_updates = (len(my_items) // (MICRO_BATCH * GRAD_ACCUM)) * EPOCHS\n",+ " scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=max(1, total_updates), eta_min=1e-6)\n"," scaler = torch.amp.GradScaler(\"cuda\", enabled=True)\n"," \n"," if rank == 0:\n",@@ -311,11 +313,11 @@" adv = r - r.mean(0, keepdim=True)\n"," adv = adv / (adv.std() + 1e-6)\n"," \n",- " # 3. Policy gradient loss + auxiliary soft cross-entropy guidance (0.5 weight)\n",+ " # 3. Policy gradient loss + full 1.0 soft cross-entropy guidance\n"," logp = -(((z - logits.unsqueeze(0)) ** 2) * mask).sum(-1) / (2 * sigma ** 2)\n"," loss_rl = -(adv * logp).mean()\n"," loss_ce = -(target * torch.log_softmax(logits.masked_fill(~mask, -1e4), -1)).sum(-1).mean()\n",- " loss = (loss_rl + 0.5 * loss_ce) / GRAD_ACCUM + 0.0 * act.sum()\n",+ " loss = (loss_rl + 1.0 * loss_ce) / GRAD_ACCUM + 0.0 * act.sum()\n"," \n"," scaler.scale(loss).backward()\n"," accum_step += 1\n",@@ -340,11 +342,11 @@"\n"," dist.barrier()\n"," \n",- " # Post-training temperature calibration on rank 0\n",+ " # Post-training temperature calibration on rank 0 (balanced across all workflows)\n"," if rank == 0:\n"," print(\"\\nFitting post-training calibration temperatures...\")\n"," model.eval()\n",- " calib_items = all_items[:150]\n",+ " calib_items = all_items[::15][:400]\n"," calib_batch = collate_train_batch(calib_items, tok.pad_token_id)\n"," with torch.no_grad():\n"," with torch.autocast(\"cuda\", dtype=torch.float16):\n",@@ -495,18 +497,18 @@" if q_type == \"choice\":\n"," keys = list(qdef[\"criteria\"].keys())\n"," pred_choice = p_ans[\"choice\"]\n",- " gold_label = g_ans[\"label\"]\n",+ " gold_label = str(g_ans[\"label\"])\n"," \n"," is_corr = float(pred_choice == gold_label)\n"," accuracies.append(is_corr)\n"," all_corrects.append(is_corr)\n",- " all_confs.append(p_ans[\"confidence\"])\n"," \n"," p_probs = np.array([p_ans[\"probabilities\"].get(k, 1e-6) for k in keys])\n"," g_probs = np.array([g_ans[\"probabilities\"].get(k, 1e-6) for k in keys])\n"," p_probs /= p_probs.sum()\n"," g_probs /= g_probs.sum()\n"," \n",+ " all_confs.append(float(p_probs.max()))\n"," soft_accuracies.append(float((p_probs * g_probs).sum()))\n"," brier_scores.append(float(((p_probs - g_probs) ** 2).sum()))\n"," tv_distances.append(float(0.5 * np.abs(p_probs - g_probs).sum()))\n",@@ -522,7 +524,7 @@" is_corr = float(pred_label == gold_label)\n"," accuracies.append(is_corr)\n"," all_corrects.append(is_corr)\n",- " all_confs.append(max(p_val, 1.0 - p_val))\n",+ " all_confs.append(float(max(p_val, 1.0 - p_val)))\n"," \n"," p_dist = np.array([1.0 - p_val, p_val])\n"," g_dist = np.array([1.0 - g_val, g_val])\n",@@ -539,12 +541,21 @@" score_maes.append(abs(p_score - g_score))\n"," within_one.append(float(abs(p_score - g_score) <= 1.0))\n"," \n",- " p_lvl = int(round(p_score))\n",+ " # Use argmax of probabilities for discrete prediction\n",+ " n_levels = len(qdef.get(\"criteria\", []))\n",+ " p_probs_score = np.array([p_ans[\"probabilities\"].get(str(i), 0.0) for i in range(n_levels)])\n",+ " if p_probs_score.sum() > 0:\n",+ " p_probs_score /= p_probs_score.sum()\n",+ " p_lvl = int(np.argmax(p_probs_score))\n",+ " all_confs.append(float(p_probs_score.max()))\n",+ " else:\n",+ " p_lvl = int(round(p_score))\n",+ " all_confs.append(0.5)\n",+ " \n"," g_lvl = int(g_ans.get(\"label\", int(round(g_score))))\n"," is_corr = float(p_lvl == g_lvl)\n"," accuracies.append(is_corr)\n"," all_corrects.append(is_corr)\n",- " all_confs.append(p_ans.get(\"confidence\", 0.5))\n","\n","laya_acc = np.mean(accuracies)\n","laya_soft_acc = np.mean(soft_accuracies)\n",