Skip to content
Python

fix: corrected EMA and added model reload training

mohamdev374b12f3e0c8
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynb
index badd8fb..94903b8 100644
--- a/notebooks/distill_transformer_decoder.ipynb
+++ b/notebooks/distill_transformer_decoder.ipynb
@@ -2,19 +2,10 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"id": "ad5fd1be",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "['TensorrtExecutionProvider', 'CUDAExecutionProvider', 'CPUExecutionProvider']\n",
- "Using device: cuda\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
@@ -54,8 +45,8 @@
"class DistillConfig:\n",
" \"\"\"All hyperparameters in one place.\"\"\"\n",
" # --- Paths ---\n",
- " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",
- " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_gt_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n",
+ " images_dir: str = str(PROJECT_ROOT / \"data/sam3d_distill_coco/images\") #path to cropped images from tools/annotate_dataset.py\n",
+ " annotations_dir: str = str(PROJECT_ROOT / \"data/sam3d_distill_coco/annotations\") #path to annotations from tools/annotate_dataset.py\n",
" # images_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/images\") #path to cropped images from tools/annotate_dataset.py\n",
" # annotations_dir: str = str(\"/home/madjel/Projects/gitpackages/sam3-biomechanics/video_to_pose_pipeline/micro_dataset_cliff/annotations\") #path to annotations from tools/annotate_dataset.py\n",
" mhr_model_path: str = str(PROJECT_ROOT / \"checkpoints/mhr_model.pt\") #path to MHR model\n",
@@ -101,7 +92,7 @@
" grad_clip: float = 5.0\n",
" use_amp: bool = True \n",
" ema_decay: float = 0.9998 # EMA shadow-weights decay (Cell 11)\n",
- " early_stop_patience: int = 30 # stop after N epochs w/o val PA-MPJPE improvement (Cell 11)\n",
+ " early_stop_patience: int = 50 # stop after N epochs where NEITHER raw nor ema improved (Cell 11)\n",
"\n",
" # --- Loss weights (UPDATED for HMR Balance) ---\n",
" w_pose: float = 1.0\n",
@@ -261,18 +252,10 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"id": "1d5e32ef",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Dataset Loaded! Train batches: 557 | Val batches: 62\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
"# Cell 3 — Dataset and Data Augmentation\n",
@@ -914,23 +897,10 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"id": "10f2586d",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--- Architecture Output Shapes ---\n",
- "mhr_params: torch.Size([2, 204])\n",
- "shape_params: torch.Size([2, 45])\n",
- "cam_trans: torch.Size([2, 3])\n",
- "joints_2d: torch.Size([2, 70, 2])\n",
- "joints_3d: torch.Size([2, 70, 3])\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
"# Cell 7 — Student Architecture (Mini-SAM3D with PE)\n",
@@ -1125,33 +1095,10 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"id": "9239175a",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--- Running Loss Function Sanity Check ---\n",
- "Losses for the Perfect Student:\n",
- " loss_pose : 0.000000\n",
- " loss_scale : 0.000000\n",
- " loss_shape : 0.000000\n",
- " loss_cam : 0.000000\n",
- " loss_mhr_joints : 0.000000\n",
- " loss_structural : 0.000000\n",
- " loss_feet : 0.000362\n",
- " loss_2d_native : 0.000000\n",
- " loss_3d_native : 0.000000\n",
- " loss_reproj_native : 0.000000\n",
- " loss_reproj_mhr : 0.000001\n",
- " total_loss : 0.000364\n",
- "\n",
- "❌ ERROR: Loss is not zero. Check the math.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
"# Cell 8 — The Distillation Loss Module (Foolproof Edition)\n",
@@ -1441,18 +1388,10 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
"id": "27616862",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "GPU allocated now: 0.74 GB\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
"# Cell 10 — HMR Evaluation Metrics (MPJPE & PA-MPJPE)\n",
@@ -1626,43 +1565,86 @@
"from tqdm.notebook import tqdm\n",
"from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn\n",
"\n",
+ "import gc, torch\n",
+ "for _n in [\"overfit_model\", \"optimizer\", \"scaler\", \"overfit_batch\", \"overfit_batch_cpu\"]:\n",
+ " globals().pop(_n, None)\n",
+ "gc.collect(); torch.cuda.empty_cache()\n",
+ "print(f\"GPU allocated now: {torch.cuda.memory_allocated()/1e9:.2f} GB\") # expect ~1.5 GB (just mhr_module)\n",
+ "\n",
"# EMA / early-stopping knobs (override in cfg if you want; sane defaults otherwise)\n",
"EMA_DECAY = getattr(cfg, \"ema_decay\", 0.9998)\n",
- "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 30) # epochs w/o val improvement\n",
+ "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 50) # epochs where NEITHER raw nor ema improved\n",
+ "\n",
+ "\n",
+ "def make_ema_avg_fn(max_decay):\n",
+ " \"\"\"EMA with a decay WARMUP: decay = min(max_decay, (1+n)/(10+n)).\n",
+ " Early on (small n) decay is small so the shadow TRACKS the model instead of\n",
+ " sitting near random init -> removes the early 'mean-pose mush' that gamed PA-MPJPE.\"\"\"\n",
+ " @torch.no_grad()\n",
+ " def avg_fn(ema_params, model_params, num_averaged):\n",
+ " n = num_averaged.item() if torch.is_tensor(num_averaged) else float(num_averaged)\n",
+ " decay = min(max_decay, (1.0 + n) / (10.0 + n))\n",
+ " torch._foreach_lerp_(ema_params, model_params, 1.0 - decay)\n",
+ " return avg_fn\n",
"\n",
"def train_instant_hmr():\n",
" print(f\"--- Training {cfg.backbone} | EMA decay={EMA_DECAY} | early-stop patience={EARLY_STOP_PATIENCE} ---\")\n",
" os.makedirs(cfg.log_dir, exist_ok=True)\n",
"\n",
- " RESUME_FROM_CHECKPOINT = False\n",
+ " # Set to True to enable resuming\n",
+ " RESUME_FROM_CHECKPOINT = True \n",
"\n",
" # 1. Model + EMA shadow + loss\n",
" model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",
- " # use_buffers=True so RepViT BatchNorm running stats are tracked by the EMA too.\n",
- " ema_model = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(EMA_DECAY), use_buffers=True)\n",
" criterion = DistillationLoss(cfg, mhr_module)\n",
"\n",
" # 2. Optimizer & AMP\n",
" optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n",
" scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",
"\n",
+ " # 3. LR schedule\n",
+ " steps_per_epoch = len(train_loader)\n",
+ " scheduler = lr_scheduler.OneCycleLR(\n",
+ " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",
+ " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n",
+ " )\n",
+ "\n",
" best_raw_path = os.path.join(cfg.log_dir, \"best_student_model_raw.pth\")\n",
" best_ema_path = os.path.join(cfg.log_dir, \"best_student_model_ema.pth\")\n",
- " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall; loaded by export/eval cells\n",
+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall\n",
+ "\n",
+ " # Tracking variables\n",
+ " start_epoch = 0\n",
+ " best_raw_pa = float('inf')\n",
+ " best_ema_pa = float('inf')\n",
+ " best_overall_pa = float('inf')\n",
+ " epochs_no_improve = 0\n",
"\n",
+ " # 4. Resume from Checkpoint Logic\n",
" if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n",
" print(f\"🔄 Resuming from checkpoint: {best_ckpt_path}\")\n",
- " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",
+ " # weights_only=False is often required to load optimizer/scheduler dicts safely in newer PyTorch versions\n",
+ " ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=False)\n",
+ " \n",
+ " # Load states\n",
" model.load_state_dict(ckpt[\"model_state_dict\"], strict=False)\n",
+ " if \"optimizer_state_dict\" in ckpt:\n",
+ " optimizer.load_state_dict(ckpt[\"optimizer_state_dict\"])\n",
+ " if \"scaler_state_dict\" in ckpt:\n",
+ " scaler.load_state_dict(ckpt[\"scaler_state_dict\"])\n",
+ " if \"scheduler_state_dict\" in ckpt:\n",
+ " scheduler.load_state_dict(ckpt[\"scheduler_state_dict\"])\n",
+ " \n",
+ " start_epoch = ckpt.get(\"epoch\", -1) + 1\n",
+ " best_overall_pa = ckpt.get(\"val_pa_mpjpe\", float('inf'))\n",
+ " best_raw_pa = best_overall_pa \n",
+ " best_ema_pa = best_overall_pa\n",
+ " print(f\"✅ Resumed successfully from Epoch {start_epoch}. Best PA-MPJPE was {best_overall_pa:.1f}mm.\")\n",
" else:\n",
" print(\"🚀 Starting training from scratch with fresh weights.\")\n",
"\n",
- " # 3. LR schedule\n",
- " steps_per_epoch = len(train_loader)\n",
- " scheduler = lr_scheduler.OneCycleLR(\n",
- " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",
- " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n",
- " )\n",
+ " # Initialize EMA *after* potential checkpoint loading so it inherits the resumed weights\n",
+ " ema_model = AveragedModel(model, multi_avg_fn=make_ema_avg_fn(EMA_DECAY), use_buffers=True)\n",
"\n",
" # --- validation helper: evaluate ANY model over val_loader (closes over criterion/loaders) ---\n",
" @torch.no_grad()\n",
@@ -1689,19 +1671,14 @@
" for k in hmr:\n",
" hmr[k] += bm.get(k, 0.0)\n",
" n += 1\n",
- " # live tqdm read-out of THIS model's running PA-MPJPE (so you see raw vs ema as they run)\n",
" vbar.set_postfix({'PA-MPJPE': f\"{hmr['Mesh_PA_MPJPE']/max(n,1):.1f}mm\"})\n",
" vbar.close()\n",
" vloss = {k: v / max(n, 1) for k, v in vloss.items()}\n",
" hmr = {k: v / max(n, 1) for k, v in hmr.items()}\n",
" return vloss, hmr, n\n",
"\n",
- " best_raw_pa = float('inf')\n",
- " best_ema_pa = float('inf')\n",
- " best_overall_pa = float('inf')\n",
- " epochs_no_improve = 0\n",
- "\n",
- " for epoch in range(cfg.epochs):\n",
+ " # Note the loop now starts at `start_epoch`\n",
+ " for epoch in range(start_epoch, cfg.epochs):\n",
" # ---------------- TRAIN ----------------\n",
" model.train()\n",
" tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
@@ -1725,9 +1702,9 @@
" scaler.step(optimizer)\n",
" scaler.update()\n",
" scale_after = scaler.get_scale()\n",
- " if scale_before <= scale_after: # optimizer actually stepped (no inf grads)\n",
+ " if scale_before <= scale_after: \n",
" scheduler.step()\n",
- " ema_model.update_parameters(model) # EMA update follows the real param step\n",
+ " ema_model.update_parameters(model) \n",
" tm['total'] += loss.item()\n",
" tm['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
" tm['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
@@ -1753,33 +1730,50 @@
" delta = raw_hmr['Mesh_PA_MPJPE'] - ema_hmr['Mesh_PA_MPJPE']\n",
" print(f\" 🔬 EMA vs RAW (Mesh PA-MPJPE): {'EMA better' if delta > 0 else 'RAW better'} by {abs(delta):.1f} mm\")\n",
"\n",
- " # ---------------- CHECKPOINTS (best of each, separately) ----------------\n",
- " if n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa:\n",
+ " # ---------------- CHECKPOINTS ----------------\n",
+ " \n",
+ " # Save standard states dict for all checkpoints to allow resuming from any of them\n",
+ " def get_save_dict(model_to_save, val_metric, source_name):\n",
+ " return {\n",
+ " 'epoch': epoch,\n",
+ " 'model_state_dict': model_to_save.state_dict(),\n",
+ " 'optimizer_state_dict': optimizer.state_dict(),\n",
+ " 'scaler_state_dict': scaler.state_dict(),\n",
+ " 'scheduler_state_dict': scheduler.state_dict(),\n",
+ " 'val_pa_mpjpe': val_metric,\n",
+ " 'source': source_name\n",
+ " }\n",
+ "\n",
+ " # ---------------- CHECKPOINTS + EARLY-STOP BOOKKEEPING ----------------\n",
+ " # Did each model improve its OWN best this epoch?\n",
+ " improved_raw = n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa\n",
+ " improved_ema = n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa\n",
+ "\n",
+ " if improved_raw:\n",
" best_raw_pa = raw_hmr['Mesh_PA_MPJPE']\n",
- " torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(),\n",
- " 'val_pa_mpjpe': best_raw_pa, 'source': 'raw'}, best_raw_path)\n",
- " if n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa:\n",
+ " torch.save(get_save_dict(model, best_raw_pa, 'raw'), best_raw_path)\n",
+ " if improved_ema:\n",
" best_ema_pa = ema_hmr['Mesh_PA_MPJPE']\n",
- " # ema_model.module is the underlying InstantHMRStudent -> clean state_dict keys\n",
- " torch.save({'epoch': epoch, 'model_state_dict': ema_model.module.state_dict(),\n",
- " 'val_pa_mpjpe': best_ema_pa, 'source': 'ema'}, best_ema_path)\n",
+ " torch.save(get_save_dict(ema_model.module, best_ema_pa, 'ema'), best_ema_path)\n",
"\n",
- " # ---------------- EARLY STOPPING (on the better of raw/EMA) ----------------\n",
+ " # Canonical \"best overall\" checkpoint (loaded by the export/eval cells)\n",
" current = min(raw_hmr['Mesh_PA_MPJPE'], ema_hmr['Mesh_PA_MPJPE'])\n",
" if current < best_overall_pa - 1e-4:\n",
" best_overall_pa = current\n",
- " epochs_no_improve = 0\n",
" use_ema = ema_hmr['Mesh_PA_MPJPE'] <= raw_hmr['Mesh_PA_MPJPE']\n",
" better = ema_model.module if use_ema else model\n",
- " torch.save({'epoch': epoch, 'model_state_dict': better.state_dict(),\n",
- " 'val_pa_mpjpe': best_overall_pa, 'source': 'ema' if use_ema else 'raw'},\n",
- " best_ckpt_path)\n",
- " print(f\"💾 New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",
+ " torch.save(get_save_dict(better, best_overall_pa, 'ema' if use_ema else 'raw'), best_ckpt_path)\n",
+ " print(f\"\\U0001f4be New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",
+ "\n",
+ " # ---------------- EARLY STOPPING ----------------\n",
+ " # Stop ONLY after EARLY_STOP_PATIENCE epochs where NEITHER raw NOR ema improved its own best.\n",
+ " if improved_raw or improved_ema:\n",
+ " epochs_no_improve = 0\n",
" else:\n",
" epochs_no_improve += 1\n",
- " print(f\" ⏳ No improvement {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best {best_overall_pa:.1f} mm)\")\n",
+ " print(f\" \\u23f3 Neither RAW nor EMA improved: {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best raw {best_raw_pa:.1f} / ema {best_ema_pa:.1f} mm)\")\n",
" if epochs_no_improve >= EARLY_STOP_PATIENCE:\n",
- " print(f\"\\n🛑 Early stopping at epoch {epoch+1}: no val improvement in {EARLY_STOP_PATIENCE} epochs.\")\n",
+ " print(f\"\\n\\U0001f6d1 Early stopping at epoch {epoch+1}: neither RAW nor EMA improved for {EARLY_STOP_PATIENCE} epochs.\")\n",
" break\n",
"\n",
" print(f\"\\n🎉 Training complete! Best overall Mesh PA-MPJPE: {best_overall_pa:.1f} mm\")\n",
@@ -1791,7 +1785,7 @@
" import gc\n",
" gc.collect()\n",
" torch.cuda.empty_cache()\n",
- " train_instant_hmr()\n"
+ " train_instant_hmr()"
]
},
{