Python
fix: corrected EMA and added model reload training
mohamdev374b12f3e0c8
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynbindex badd8fb..94903b8 100644--- a/notebooks/distill_transformer_decoder.ipynb+++ b/notebooks/distill_transformer_decoder.ipynb@@ -2,19 +2,10 @@"cells": [{"cell_type": "code",- "execution_count": 3,+ "execution_count": null,"id": "ad5fd1be","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']\n",- "Using device: cuda\n"- ]- }- ],+ "outputs": [],"source": ["import torch\n","import torch.nn as nn\n",@@ -54,8 +45,8 @@"class DistillConfig:\n"," \"\"\"All hyperparameters in one place.\"\"\"\n"," # --- Paths ---\n",- " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",- " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n",+ " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_distill_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",+ " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_distill_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n"," # images_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n"," # annotations_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n"," mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/mhr_model.pt\") #path to MHR model\n",@@ -101,7 +92,7 @@" grad_clip: float = 5.0\n"," use_amp: bool = True \n"," ema_decay: float = 0.9998 # EMA shadow-weights decay (Cell 11)\n",- " early_stop_patience: int = 30 # stop after N epochs w/o val PA-MPJPE improvement (Cell 11)\n",+ " early_stop_patience: int = 50 # stop after N epochs where NEITHER raw nor ema improved (Cell 11)\n","\n"," # --- Loss weights (UPDATED for HMR Balance) ---\n"," w_pose: float = 1.0\n",@@ -261,18 +252,10 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": null,"id": "1d5e32ef","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Dataset Loaded! Train batches: 557 | Val batches: 62\n"- ]- }- ],+ "outputs": [],"source": ["# ============================================================\n","# Cell 3 — Dataset and Data Augmentation\n",@@ -914,23 +897,10 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": null,"id": "10f2586d","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "--- Architecture Output Shapes ---\n",- "mhr_params: torch.Size([2, 204])\n",- "shape_params: torch.Size([2, 45])\n",- "cam_trans: torch.Size([2, 3])\n",- "joints_2d: torch.Size([2, 70, 2])\n",- "joints_3d: torch.Size([2, 70, 3])\n"- ]- }- ],+ "outputs": [],"source": ["# ============================================================\n","# Cell 7 — Student Architecture (Mini-SAM3D with PE)\n",@@ -1125,33 +1095,10 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": null,"id": "9239175a","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "--- Running Loss Function Sanity Check ---\n",- "Losses for the Perfect Student:\n",- " loss_pose : 0.000000\n",- " loss_scale : 0.000000\n",- " loss_shape : 0.000000\n",- " loss_cam : 0.000000\n",- " loss_mhr_joints : 0.000000\n",- " loss_structural : 0.000000\n",- " loss_feet : 0.000362\n",- " loss_2d_native : 0.000000\n",- " loss_3d_native : 0.000000\n",- " loss_reproj_native : 0.000000\n",- " loss_reproj_mhr : 0.000001\n",- " total_loss : 0.000364\n",- "\n",- "❌ ERROR: Loss is not zero. Check the math.\n"- ]- }- ],+ "outputs": [],"source": ["# ============================================================\n","# Cell 8 — The Distillation Loss Module (Foolproof Edition)\n",@@ -1441,18 +1388,10 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": null,"id": "27616862","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "GPU allocated now: 0.74 GB\n"- ]- }- ],+ "outputs": [],"source": ["# ============================================================\n","# Cell 10 — HMR Evaluation Metrics (MPJPE & PA-MPJPE)\n",@@ -1626,43 +1565,86 @@"from tqdm.notebook import tqdm\n","from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn\n","\n",+ "import gc, torch\n",+ "for _n in [\"overfit_model\", \"optimizer\", \"scaler\", \"overfit_batch\", \"overfit_batch_cpu\"]:\n",+ " globals().pop(_n, None)\n",+ "gc.collect(); torch.cuda.empty_cache()\n",+ "print(f\"GPU allocated now: {torch.cuda.memory_allocated()/1e9:.2f} GB\") # expect ~1.5 GB (just mhr_module)\n",+ "\n","# EMA / early-stopping knobs (override in cfg if you want; sane defaults otherwise)\n","EMA_DECAY = getattr(cfg, \"ema_decay\", 0.9998)\n",- "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 30) # epochs w/o val improvement\n",+ "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 50) # epochs where NEITHER raw nor ema improved\n",+ "\n",+ "\n",+ "def make_ema_avg_fn(max_decay):\n",+ " \"\"\"EMA with a decay WARMUP: decay = min(max_decay, (1+n)/(10+n)).\n",+ " Early on (small n) decay is small so the shadow TRACKS the model instead of\n",+ " sitting near random init -> removes the early 'mean-pose mush' that gamed PA-MPJPE.\"\"\"\n",+ " @torch.no_grad()\n",+ " def avg_fn(ema_params, model_params, num_averaged):\n",+ " n = num_averaged.item() if torch.is_tensor(num_averaged) else float(num_averaged)\n",+ " decay = min(max_decay, (1.0 + n) / (10.0 + n))\n",+ " torch._foreach_lerp_(ema_params, model_params, 1.0 - decay)\n",+ " return avg_fn\n","\n","def train_instant_hmr():\n"," print(f\"--- Training {cfg.backbone} | EMA decay={EMA_DECAY} | early-stop patience={EARLY_STOP_PATIENCE} ---\")\n"," os.makedirs(cfg.log_dir, exist_ok=True)\n","\n",- " RESUME_FROM_CHECKPOINT = False\n",+ " # Set to True to enable resuming\n",+ " RESUME_FROM_CHECKPOINT = True \n","\n"," # 1. Model + EMA shadow + loss\n"," model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",- " # use_buffers=True so RepViT BatchNorm running stats are tracked by the EMA too.\n",- " ema_model = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(EMA_DECAY), use_buffers=True)\n"," criterion = DistillationLoss(cfg, mhr_module)\n","\n"," # 2. Optimizer & AMP\n"," optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n"," scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n","\n",+ " # 3. LR schedule\n",+ " steps_per_epoch = len(train_loader)\n",+ " scheduler = lr_scheduler.OneCycleLR(\n",+ " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",+ " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n",+ " )\n",+ "\n"," best_raw_path = os.path.join(cfg.log_dir, \"best_student_model_raw.pth\")\n"," best_ema_path = os.path.join(cfg.log_dir, \"best_student_model_ema.pth\")\n",- " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall; loaded by export/eval cells\n",+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall\n",+ "\n",+ " # Tracking variables\n",+ " start_epoch = 0\n",+ " best_raw_pa = float('inf')\n",+ " best_ema_pa = float('inf')\n",+ " best_overall_pa = float('inf')\n",+ " epochs_no_improve = 0\n","\n",+ " # 4. Resume from Checkpoint Logic\n"," if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n"," print(f\"🔄 Resuming from checkpoint: {best_ckpt_path}\")\n",- " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",+ " # weights_only=False is often required to load optimizer/scheduler dicts safely in newer PyTorch versions\n",+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=False)\n",+ " \n",+ " # Load states\n"," model.load_state_dict(ckpt[\"model_state_dict\"], strict=False)\n",+ " if \"optimizer_state_dict\" in ckpt:\n",+ " optimizer.load_state_dict(ckpt[\"optimizer_state_dict\"])\n",+ " if \"scaler_state_dict\" in ckpt:\n",+ " scaler.load_state_dict(ckpt[\"scaler_state_dict\"])\n",+ " if \"scheduler_state_dict\" in ckpt:\n",+ " scheduler.load_state_dict(ckpt[\"scheduler_state_dict\"])\n",+ " \n",+ " start_epoch = ckpt.get(\"epoch\", -1) + 1\n",+ " best_overall_pa = ckpt.get(\"val_pa_mpjpe\", float('inf'))\n",+ " best_raw_pa = best_overall_pa \n",+ " best_ema_pa = best_overall_pa\n",+ " print(f\"✅ Resumed successfully from Epoch {start_epoch}. Best PA-MPJPE was {best_overall_pa:.1f}mm.\")\n"," else:\n"," print(\"🚀 Starting training from scratch with fresh weights.\")\n","\n",- " # 3. LR schedule\n",- " steps_per_epoch = len(train_loader)\n",- " scheduler = lr_scheduler.OneCycleLR(\n",- " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",- " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n",- " )\n",+ " # Initialize EMA *after* potential checkpoint loading so it inherits the resumed weights\n",+ " ema_model = AveragedModel(model, multi_avg_fn=make_ema_avg_fn(EMA_DECAY), use_buffers=True)\n","\n"," # --- validation helper: evaluate ANY model over val_loader (closes over criterion/loaders) ---\n"," @torch.no_grad()\n",@@ -1689,19 +1671,14 @@" for k in hmr:\n"," hmr[k] += bm.get(k, 0.0)\n"," n += 1\n",- " # live tqdm read-out of THIS model's running PA-MPJPE (so you see raw vs ema as they run)\n"," vbar.set_postfix({'PA-MPJPE': f\"{hmr['Mesh_PA_MPJPE']/max(n,1):.1f}mm\"})\n"," vbar.close()\n"," vloss = {k: v / max(n, 1) for k, v in vloss.items()}\n"," hmr = {k: v / max(n, 1) for k, v in hmr.items()}\n"," return vloss, hmr, n\n","\n",- " best_raw_pa = float('inf')\n",- " best_ema_pa = float('inf')\n",- " best_overall_pa = float('inf')\n",- " epochs_no_improve = 0\n",- "\n",- " for epoch in range(cfg.epochs):\n",+ " # Note the loop now starts at `start_epoch`\n",+ " for epoch in range(start_epoch, cfg.epochs):\n"," # ---------------- TRAIN ----------------\n"," model.train()\n"," tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",@@ -1725,9 +1702,9 @@" scaler.step(optimizer)\n"," scaler.update()\n"," scale_after = scaler.get_scale()\n",- " if scale_before <= scale_after: # optimizer actually stepped (no inf grads)\n",+ " if scale_before <= scale_after: \n"," scheduler.step()\n",- " ema_model.update_parameters(model) # EMA update follows the real param step\n",+ " ema_model.update_parameters(model) \n"," tm['total'] += loss.item()\n"," tm['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n"," tm['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",@@ -1753,33 +1730,50 @@" delta = raw_hmr['Mesh_PA_MPJPE'] - ema_hmr['Mesh_PA_MPJPE']\n"," print(f\" 🔬 EMA vs RAW (Mesh PA-MPJPE): {'EMA better' if delta > 0 else 'RAW better'} by {abs(delta):.1f} mm\")\n","\n",- " # ---------------- CHECKPOINTS (best of each, separately) ----------------\n",- " if n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa:\n",+ " # ---------------- CHECKPOINTS ----------------\n",+ " \n",+ " # Save standard states dict for all checkpoints to allow resuming from any of them\n",+ " def get_save_dict(model_to_save, val_metric, source_name):\n",+ " return {\n",+ " 'epoch': epoch,\n",+ " 'model_state_dict': model_to_save.state_dict(),\n",+ " 'optimizer_state_dict': optimizer.state_dict(),\n",+ " 'scaler_state_dict': scaler.state_dict(),\n",+ " 'scheduler_state_dict': scheduler.state_dict(),\n",+ " 'val_pa_mpjpe': val_metric,\n",+ " 'source': source_name\n",+ " }\n",+ "\n",+ " # ---------------- CHECKPOINTS + EARLY-STOP BOOKKEEPING ----------------\n",+ " # Did each model improve its OWN best this epoch?\n",+ " improved_raw = n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa\n",+ " improved_ema = n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa\n",+ "\n",+ " if improved_raw:\n"," best_raw_pa = raw_hmr['Mesh_PA_MPJPE']\n",- " torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(),\n",- " 'val_pa_mpjpe': best_raw_pa, 'source': 'raw'}, best_raw_path)\n",- " if n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa:\n",+ " torch.save(get_save_dict(model, best_raw_pa, 'raw'), best_raw_path)\n",+ " if improved_ema:\n"," best_ema_pa = ema_hmr['Mesh_PA_MPJPE']\n",- " # ema_model.module is the underlying InstantHMRStudent -> clean state_dict keys\n",- " torch.save({'epoch': epoch, 'model_state_dict': ema_model.module.state_dict(),\n",- " 'val_pa_mpjpe': best_ema_pa, 'source': 'ema'}, best_ema_path)\n",+ " torch.save(get_save_dict(ema_model.module, best_ema_pa, 'ema'), best_ema_path)\n","\n",- " # ---------------- EARLY STOPPING (on the better of raw/EMA) ----------------\n",+ " # Canonical \"best overall\" checkpoint (loaded by the export/eval cells)\n"," current = min(raw_hmr['Mesh_PA_MPJPE'], ema_hmr['Mesh_PA_MPJPE'])\n"," if current < best_overall_pa - 1e-4:\n"," best_overall_pa = current\n",- " epochs_no_improve = 0\n"," use_ema = ema_hmr['Mesh_PA_MPJPE'] <= raw_hmr['Mesh_PA_MPJPE']\n"," better = ema_model.module if use_ema else model\n",- " torch.save({'epoch': epoch, 'model_state_dict': better.state_dict(),\n",- " 'val_pa_mpjpe': best_overall_pa, 'source': 'ema' if use_ema else 'raw'},\n",- " best_ckpt_path)\n",- " print(f\"💾 New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",+ " torch.save(get_save_dict(better, best_overall_pa, 'ema' if use_ema else 'raw'), best_ckpt_path)\n",+ " print(f\"\\U0001f4be New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",+ "\n",+ " # ---------------- EARLY STOPPING ----------------\n",+ " # Stop ONLY after EARLY_STOP_PATIENCE epochs where NEITHER raw NOR ema improved its own best.\n",+ " if improved_raw or improved_ema:\n",+ " epochs_no_improve = 0\n"," else:\n"," epochs_no_improve += 1\n",- " print(f\" ⏳ No improvement {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best {best_overall_pa:.1f} mm)\")\n",+ " print(f\" \\u23f3 Neither RAW nor EMA improved: {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best raw {best_raw_pa:.1f} / ema {best_ema_pa:.1f} mm)\")\n"," if epochs_no_improve >= EARLY_STOP_PATIENCE:\n",- " print(f\"\\n🛑 Early stopping at epoch {epoch+1}: no val improvement in {EARLY_STOP_PATIENCE} epochs.\")\n",+ " print(f\"\\n\\U0001f6d1 Early stopping at epoch {epoch+1}: neither RAW nor EMA improved for {EARLY_STOP_PATIENCE} epochs.\")\n"," break\n","\n"," print(f\"\\n🎉 Training complete! Best overall Mesh PA-MPJPE: {best_overall_pa:.1f} mm\")\n",@@ -1791,7 +1785,7 @@" import gc\n"," gc.collect()\n"," torch.cuda.empty_cache()\n",- " train_instant_hmr()\n"+ " train_instant_hmr()"]},{