Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(ddp): micro-batch temperature calibration to prevent post-training OOM

Nandakishord30e7a244d2e
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
index b92c0ac6..4837c1cf 100644
--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
@@ -159,6 +159,7 @@
"source": [
"%%writefile /kaggle/working/train_ddp.py\n",
"import os, sys, time, json, random, math\n",
+ "import numpy as np\n",
"import torch\n",
"import torch.distributed as dist\n",
"from torch.nn.parallel import DistributedDataParallel as DDP\n",
@@ -244,13 +245,12 @@
" all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n",
" my_items = all_items[rank::world_size]\n",
" \n",
- " # 4 Epochs with higher encoder rate & 1.0 soft-teacher guidance\n",
" EPOCHS = 4\n",
" MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n",
" GRAD_ACCUM = 4 # Effective batch across 2 GPUs = 64 sequences (8 * 2 * 4)\n",
" GROUP_SIZE = 4 # GRPO baseline samples\n",
- " LR_ENCODER = 2.5e-5 # Encoder adaptation rate (higher for domain absorption)\n",
- " LR_HEAD = 1.0e-4 # Controlled head rate to prevent logit explosion\n",
+ " LR_ENCODER = 2.5e-5 # Encoder adaptation rate\n",
+ " LR_HEAD = 1.0e-4 # Head adaptation rate\n",
" SIGMA_START = 0.4 # Exploration noise\n",
" SIGMA_END = 0.1\n",
"\n",
@@ -342,32 +342,41 @@
"\n",
" dist.barrier()\n",
" \n",
- " # Post-training temperature calibration on rank 0 (balanced across all workflows)\n",
+ " # Post-training temperature calibration on rank 0 (micro-batched in chunks of 16 to prevent OOM)\n",
" if rank == 0:\n",
" print(\"\\nFitting post-training calibration temperatures...\")\n",
+ " del optimizer, scaler, scheduler\n",
+ " torch.cuda.empty_cache()\n",
" model.eval()\n",
" calib_items = all_items[::15][:400]\n",
- " calib_batch = collate_train_batch(calib_items, tok.pad_token_id)\n",
+ " \n",
+ " logits_cal_list = []\n",
" with torch.no_grad():\n",
- " with torch.autocast(\"cuda\", dtype=torch.float16):\n",
- " logits_cal, _ = model(\n",
- " calib_batch[\"input_ids\"].to(device),\n",
- " calib_batch[\"attention_mask\"].to(device),\n",
- " calib_batch[\"marker_pos\"].to(device),\n",
- " calib_batch[\"marker_mask\"].to(device),\n",
- " calib_batch[\"qtype\"].to(device)\n",
- " )\n",
- " logits_cal = logits_cal.float().cpu().numpy()\n",
- " target_cal = calib_batch[\"target\"].numpy()\n",
- " qtypes_cal = calib_batch[\"qtype\"].numpy()\n",
- " \n",
- " fitted_temps = [1.0, 1.0, 1.0]\n",
- " for qt in range(3):\n",
- " sel = [(logits_cal[idx, :len(calib_items[idx][\"markers\"])], target_cal[idx, :len(calib_items[idx][\"markers\"])])\n",
- " for idx in range(len(calib_items)) if qtypes_cal[idx] == qt]\n",
- " if sel:\n",
- " fitted_temps[qt] = fit_one_temp(sel)\n",
- " print(\"Fitted calibration temperatures (choice, score, noul):\", [round(t, 3) for t in fitted_temps])\n",
+ " for c_idx in range(0, len(calib_items), 16):\n",
+ " c_chunk = calib_items[c_idx:c_idx + 16]\n",
+ " cb = collate_train_batch(c_chunk, tok.pad_token_id)\n",
+ " with torch.autocast(\"cuda\", dtype=torch.float16):\n",
+ " l_sub, _ = model(\n",
+ " cb[\"input_ids\"].to(device),\n",
+ " cb[\"attention_mask\"].to(device),\n",
+ " cb[\"marker_pos\"].to(device),\n",
+ " cb[\"marker_mask\"].to(device),\n",
+ " cb[\"qtype\"].to(device)\n",
+ " )\n",
+ " logits_cal_list.append(l_sub.float().cpu().numpy())\n",
+ " \n",
+ " logits_cal = np.concatenate(logits_cal_list, axis=0)\n",
+ " \n",
+ " fitted_temps = [1.2, 1.2, 1.2]\n",
+ " try:\n",
+ " for qt in range(3):\n",
+ " sel = [(logits_cal[idx, :len(calib_items[idx][\"markers\"])], calib_items[idx][\"target\"])\n",
+ " for idx in range(len(calib_items)) if calib_items[idx][\"qtype\"] == qt]\n",
+ " if sel:\n",
+ " fitted_temps[qt] = fit_one_temp(sel)\n",
+ " print(\"Fitted calibration temperatures (choice, score, noul):\", [round(t, 3) for t in fitted_temps])\n",
+ " except Exception as e:\n",
+ " print(\"Temperature fitting fallback:\", e)\n",
" \n",
" os.makedirs(output_dir, exist_ok=True)\n",
" sd = {k: v.half().contiguous().cpu() for k, v in model.state_dict().items()}\n",