Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(ddp): micro-batch temperature calibration to prevent post-training OOM
Nandakishord30e7a244d2e
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynbindex b92c0ac6..4837c1cf 100644--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb@@ -159,6 +159,7 @@"source": ["%%writefile /kaggle/working/train_ddp.py\n","import os, sys, time, json, random, math\n",+ "import numpy as np\n","import torch\n","import torch.distributed as dist\n","from torch.nn.parallel import DistributedDataParallel as DDP\n",@@ -244,13 +245,12 @@" all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n"," my_items = all_items[rank::world_size]\n"," \n",- " # 4 Epochs with higher encoder rate & 1.0 soft-teacher guidance\n"," EPOCHS = 4\n"," MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n"," GRAD_ACCUM = 4 # Effective batch across 2 GPUs = 64 sequences (8 * 2 * 4)\n"," GROUP_SIZE = 4 # GRPO baseline samples\n",- " LR_ENCODER = 2.5e-5 # Encoder adaptation rate (higher for domain absorption)\n",- " LR_HEAD = 1.0e-4 # Controlled head rate to prevent logit explosion\n",+ " LR_ENCODER = 2.5e-5 # Encoder adaptation rate\n",+ " LR_HEAD = 1.0e-4 # Head adaptation rate\n"," SIGMA_START = 0.4 # Exploration noise\n"," SIGMA_END = 0.1\n","\n",@@ -342,32 +342,41 @@"\n"," dist.barrier()\n"," \n",- " # Post-training temperature calibration on rank 0 (balanced across all workflows)\n",+ " # Post-training temperature calibration on rank 0 (micro-batched in chunks of 16 to prevent OOM)\n"," if rank == 0:\n"," print(\"\\nFitting post-training calibration temperatures...\")\n",+ " del optimizer, scaler, scheduler\n",+ " torch.cuda.empty_cache()\n"," model.eval()\n"," calib_items = all_items[::15][:400]\n",- " calib_batch = collate_train_batch(calib_items, tok.pad_token_id)\n",+ " \n",+ " logits_cal_list = []\n"," with torch.no_grad():\n",- " with torch.autocast(\"cuda\", dtype=torch.float16):\n",- " logits_cal, _ = model(\n",- " calib_batch[\"input_ids\"].to(device),\n",- " calib_batch[\"attention_mask\"].to(device),\n",- " calib_batch[\"marker_pos\"].to(device),\n",- " calib_batch[\"marker_mask\"].to(device),\n",- " calib_batch[\"qtype\"].to(device)\n",- " )\n",- " logits_cal = logits_cal.float().cpu().numpy()\n",- " target_cal = calib_batch[\"target\"].numpy()\n",- " qtypes_cal = calib_batch[\"qtype\"].numpy()\n",- " \n",- " fitted_temps = [1.0, 1.0, 1.0]\n",- " for qt in range(3):\n",- " sel = [(logits_cal[idx, :len(calib_items[idx][\"markers\"])], target_cal[idx, :len(calib_items[idx][\"markers\"])])\n",- " for idx in range(len(calib_items)) if qtypes_cal[idx] == qt]\n",- " if sel:\n",- " fitted_temps[qt] = fit_one_temp(sel)\n",- " print(\"Fitted calibration temperatures (choice, score, noul):\", [round(t, 3) for t in fitted_temps])\n",+ " for c_idx in range(0, len(calib_items), 16):\n",+ " c_chunk = calib_items[c_idx:c_idx + 16]\n",+ " cb = collate_train_batch(c_chunk, tok.pad_token_id)\n",+ " with torch.autocast(\"cuda\", dtype=torch.float16):\n",+ " l_sub, _ = model(\n",+ " cb[\"input_ids\"].to(device),\n",+ " cb[\"attention_mask\"].to(device),\n",+ " cb[\"marker_pos\"].to(device),\n",+ " cb[\"marker_mask\"].to(device),\n",+ " cb[\"qtype\"].to(device)\n",+ " )\n",+ " logits_cal_list.append(l_sub.float().cpu().numpy())\n",+ " \n",+ " logits_cal = np.concatenate(logits_cal_list, axis=0)\n",+ " \n",+ " fitted_temps = [1.2, 1.2, 1.2]\n",+ " try:\n",+ " for qt in range(3):\n",+ " sel = [(logits_cal[idx, :len(calib_items[idx][\"markers\"])], calib_items[idx][\"target\"])\n",+ " for idx in range(len(calib_items)) if calib_items[idx][\"qtype\"] == qt]\n",+ " if sel:\n",+ " fitted_temps[qt] = fit_one_temp(sel)\n",+ " print(\"Fitted calibration temperatures (choice, score, noul):\", [round(t, 3) for t in fitted_temps])\n",+ " except Exception as e:\n",+ " print(\"Temperature fitting fallback:\", e)\n"," \n"," os.makedirs(output_dir, exist_ok=True)\n"," sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}\n",