Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(finetune): hold the calibration slice out of training (#191)

Andrew Zagoryanskiyef90a4311b35
diff --git a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
index 21740f3a..c9c51fe8 100644
--- a/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
+++ b/notebooks/laya_finetune_typed_decisions_2xT4_kaggle.ipynb
@@ -249,7 +249,19 @@
" ddp_model = DDP(model, device_ids=[local_rank], find_unused_parameters=True)\n",
" \n",
" all_items = torch.load(\"/kaggle/working/train_items.pt\", weights_only=False)\n",
- " my_items = all_items[rank::world_size]\n",
+ "\n",
+ " # Hold the calibration slice out of training before sharding. Temperatures fitted on\n",
+ " # items the run has already trained on measure the fit rather than the calibration: the\n",
+ " # model is near-certain and near-correct on them, so the optimiser has nothing to soften\n",
+ " # and returns a degenerate scale. The seed is fixed and rank-independent, so every rank\n",
+ " # withholds exactly the same items and none of them reaches a training batch.\n",
+ " CALIB_MAX = 400\n",
+ " order = list(range(len(all_items)))\n",
+ " random.Random(20260922).shuffle(order)\n",
+ " n_calib = min(CALIB_MAX, len(all_items) // 10)\n",
+ " calib_items = [all_items[i] for i in sorted(order[:n_calib])]\n",
+ " train_items = [all_items[i] for i in sorted(order[n_calib:])]\n",
+ " my_items = train_items[rank::world_size]\n",
" \n",
" EPOCHS = 4\n",
" MICRO_BATCH = 8 # 8 sequences per forward pass per GPU\n",
@@ -273,7 +285,7 @@
" scaler = torch.amp.GradScaler(\"cuda\", enabled=True)\n",
" \n",
" if rank == 0:\n",
- " print(f\"Starting 2xT4 DDP training: {len(all_items)} total items | {len(my_items)} per rank | {EPOCHS} epochs\")\n",
+ " print(f\"Starting 2xT4 DDP training: {len(train_items)} train items ({len(calib_items)} held out for calibration) | {len(my_items)} per rank | {EPOCHS} epochs\")\n",
" t0 = time.time()\n",
" \n",
" for epoch in range(EPOCHS):\n",
@@ -373,7 +385,7 @@
" del optimizer, scaler, scheduler\n",
" torch.cuda.empty_cache()\n",
" model.eval()\n",
- " calib_items = all_items[::15][:400]\n",
+ " # calib_items was held out above and never entered a training batch\n",
" calib_preds = []\n",
" with torch.no_grad():\n",
" for c_idx in range(0, len(calib_items), 16):\n",