Python
feat: added exponential moving average on model weights for regularization
mohamdev885594f768b2
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynbindex 2745eac..badd8fb 100644--- a/notebooks/distill_transformer_decoder.ipynb+++ b/notebooks/distill_transformer_decoder.ipynb@@ -37,22 +37,10 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": null,"id": "7e2ae437","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Project root: /home/madjel/Projects/gitpackages/sam3d-student-release\n",- "Images dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/images\n",- "Annotations dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/annotations\n",- "MHR model path: /home/madjel/Projects/gitpackages/sam3d-student-release/checkpoints/mhr_model.pt\n",- "Config loaded. Backbone: repvit_m2_3. Ready for V3 Training.\n"- ]- }- ],+ "outputs": [],"source": ["# ============================================================\n","# Cell 1 β Configuration\n",@@ -112,6 +100,8 @@" warmup_epochs: int = 3\n"," grad_clip: float = 5.0\n"," use_amp: bool = True \n",+ " ema_decay: float = 0.9998 # EMA shadow-weights decay (Cell 11)\n",+ " early_stop_patience: int = 30 # stop after N epochs w/o val PA-MPJPE improvement (Cell 11)\n","\n"," # --- Loss weights (UPDATED for HMR Balance) ---\n"," w_pose: float = 1.0\n",@@ -1623,1095 +1613,10 @@"execution_count": null,"id": "d44aeb4a","metadata": {},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "--- Starting Full Training Pipeline: repvit_m2_3 ---\n"- ]- },- {- "name": "stderr",- "output_type": "stream",- "text": [- "Unexpected keys (head.head.bn.num_batches_tracked, head.head.bn.bias, head.head.bn.running_mean, head.head.bn.running_var, head.head.bn.weight, head.head_dist.bn.num_batches_tracked, head.head_dist.bn.bias, head.head_dist.bn.running_mean, head.head_dist.bn.running_var, head.head_dist.bn.weight) found while loading pretrained weights. This may be expected if model is being adapted.\n"- ]- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "π Starting training from scratch with fresh weights.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "87ac1c75193b444f90e0f56c736cb2c7",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 1/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "803af2b02f5e4ceda96daee5032a7cf9",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 1/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 1 Summary | LR: 1.28e-05\n",- " [Train Loss] Tot: 1.7388 | MHR: 0.0153 | Struct: 0.0003 | 2D: 0.7564 | 3D: 0.6079\n",- " [Val Loss] Tot: 1.5152 | MHR: 0.0149 | Struct: 0.0002 | 2D: 0.7518 | 3D: 0.4873\n",- " π [Val Metrics] Mesh PA-MPJPE: 179.5 mm | Mesh MPJPE: 268.8 mm\n",- " π [Val Metrics] Head PA-MPJPE: 305.6 mm | Head MPJPE: 604.1 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 179.5 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "3020c4fe8796467ebc911f875a2b947e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 2/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "19f6af9412c6427c8a8d6915ebedf21f",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 2/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 2 Summary | LR: 1.51e-05\n",- " [Train Loss] Tot: 1.4542 | MHR: 0.0149 | Struct: 0.0002 | 2D: 0.7299 | 3D: 0.4811\n",- " [Val Loss] Tot: 1.4444 | MHR: 0.0148 | Struct: 0.0001 | 2D: 0.7298 | 3D: 0.4807\n",- " π [Val Metrics] Mesh PA-MPJPE: 188.0 mm | Mesh MPJPE: 265.7 mm\n",- " π [Val Metrics] Head PA-MPJPE: 273.5 mm | Head MPJPE: 602.0 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "fe0d1a4cabba48eab7ef2c4a60371c3b",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 3/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "28c4e59d28444e9aa3e6dad69996539d",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 3/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 3 Summary | LR: 1.90e-05\n",- " [Train Loss] Tot: 1.2657 | MHR: 0.0148 | Struct: 0.0001 | 2D: 0.6214 | 3D: 0.4171\n",- " [Val Loss] Tot: 1.0030 | MHR: 0.0149 | Struct: 0.0001 | 2D: 0.4181 | 3D: 0.3143\n",- " π [Val Metrics] Mesh PA-MPJPE: 164.4 mm | Mesh MPJPE: 279.0 mm\n",- " π [Val Metrics] Head PA-MPJPE: 268.2 mm | Head MPJPE: 361.1 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 164.4 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "3e1f88f6fd344f48b7cc1d2c7f2f2e11",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 4/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "7e99577ffdf942e48c581b14cafcdb6b",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 4/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 4 Summary | LR: 2.44e-05\n",- " [Train Loss] Tot: 0.9220 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.4073 | 3D: 0.3005\n",- " [Val Loss] Tot: 0.9751 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.4150 | 3D: 0.2991\n",- " π [Val Metrics] Mesh PA-MPJPE: 188.7 mm | Mesh MPJPE: 266.9 mm\n",- " π [Val Metrics] Head PA-MPJPE: 265.9 mm | Head MPJPE: 350.1 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "5e00a1d07f274859af01b22b16f17dd9",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 5/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "f585f7a075344ffc94155af8c0db1d77",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 5/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 5 Summary | LR: 3.12e-05\n",- " [Train Loss] Tot: 0.8757 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.3762 | 3D: 0.2941\n",- " [Val Loss] Tot: 0.8734 | MHR: 0.0145 | Struct: 0.0001 | 2D: 0.3690 | 3D: 0.2949\n",- " π [Val Metrics] Mesh PA-MPJPE: 167.5 mm | Mesh MPJPE: 273.3 mm\n",- " π [Val Metrics] Head PA-MPJPE: 244.7 mm | Head MPJPE: 345.8 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "dc32d42921644e9481bc743c538c7486",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 6/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "ac683d5f8b7043248b6cbaace330fdc5",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 6/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 6 Summary | LR: 3.94e-05\n",- " [Train Loss] Tot: 0.8386 | MHR: 0.0145 | Struct: 0.0001 | 2D: 0.3555 | 3D: 0.2861\n",- " [Val Loss] Tot: 0.8797 | MHR: 0.0144 | Struct: 0.0001 | 2D: 0.3649 | 3D: 0.2821\n",- " π [Val Metrics] Mesh PA-MPJPE: 174.5 mm | Mesh MPJPE: 269.4 mm\n",- " π [Val Metrics] Head PA-MPJPE: 236.2 mm | Head MPJPE: 337.8 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d3e192c380ac40bd9893dc54989cd764",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 7/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "0c8f9fef643448c3b7d28719335d4726",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 7/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 7 Summary | LR: 4.89e-05\n",- " [Train Loss] Tot: 0.8004 | MHR: 0.0144 | Struct: 0.0001 | 2D: 0.3354 | 3D: 0.2764\n",- " [Val Loss] Tot: 0.8025 | MHR: 0.0137 | Struct: 0.0000 | 2D: 0.3330 | 3D: 0.2679\n",- " π [Val Metrics] Mesh PA-MPJPE: 174.4 mm | Mesh MPJPE: 259.5 mm\n",- " π [Val Metrics] Head PA-MPJPE: 219.4 mm | Head MPJPE: 330.2 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "a821d048197346e599af1307b601d1ed",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 8/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "986d9d359f7f43c590976f964bba2fc9",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 8/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 8 Summary | LR: 5.95e-05\n",- " [Train Loss] Tot: 0.7231 | MHR: 0.0136 | Struct: 0.0001 | 2D: 0.2984 | 3D: 0.2468\n",- " [Val Loss] Tot: 4.0354 | MHR: 0.0138 | Struct: 0.0002 | 2D: 0.3756 | 3D: 0.3284\n",- " π [Val Metrics] Mesh PA-MPJPE: 155.2 mm | Mesh MPJPE: 258.9 mm\n",- " π [Val Metrics] Head PA-MPJPE: 215.3 mm | Head MPJPE: 326.2 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 155.2 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d5b710c460904c0ca501d007102a5e88",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 9/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "0bff56f501b945f89aed52c07a4799c4",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 9/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 9 Summary | LR: 7.13e-05\n",- " [Train Loss] Tot: 0.6247 | MHR: 0.0126 | Struct: 0.0001 | 2D: 0.2521 | 3D: 0.2064\n",- " [Val Loss] Tot: 0.6047 | MHR: 0.0122 | Struct: 0.0001 | 2D: 0.2531 | 3D: 0.1814\n",- " π [Val Metrics] Mesh PA-MPJPE: 171.1 mm | Mesh MPJPE: 232.1 mm\n",- " π [Val Metrics] Head PA-MPJPE: 193.6 mm | Head MPJPE: 290.4 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "2421f23dd24e4a97b67ec7e52b9fbb7b",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 10/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "5f62154c7c3b44518cd53afc2fe56253",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 10/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 10 Summary | LR: 8.39e-05\n",- " [Train Loss] Tot: 0.5327 | MHR: 0.0117 | Struct: 0.0001 | 2D: 0.2063 | 3D: 0.1734\n",- " [Val Loss] Tot: 0.5420 | MHR: 0.0113 | Struct: 0.0001 | 2D: 0.1963 | 3D: 0.1619\n",- " π [Val Metrics] Mesh PA-MPJPE: 134.2 mm | Mesh MPJPE: 227.0 mm\n",- " π [Val Metrics] Head PA-MPJPE: 169.2 mm | Head MPJPE: 270.3 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 134.2 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "e7f7207d3ae84a0582c9550e50aab0d8",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 11/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "26284798653b49c2b72f127be1570e5d",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 11/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 11 Summary | LR: 9.73e-05\n",- " [Train Loss] Tot: 0.4547 | MHR: 0.0109 | Struct: 0.0001 | 2D: 0.1685 | 3D: 0.1470\n",- " [Val Loss] Tot: 0.4793 | MHR: 0.0104 | Struct: 0.0001 | 2D: 0.1884 | 3D: 0.1442\n",- " π [Val Metrics] Mesh PA-MPJPE: 145.8 mm | Mesh MPJPE: 210.3 mm\n",- " π [Val Metrics] Head PA-MPJPE: 159.2 mm | Head MPJPE: 255.8 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "7b2f8d8116544c66af0d55a98c45baaa",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 12/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "9928b32d7dea4e019765c3698cec5304",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 12/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 12 Summary | LR: 1.11e-04\n",- " [Train Loss] Tot: 0.3963 | MHR: 0.0104 | Struct: 0.0001 | 2D: 0.1397 | 3D: 0.1282\n",- " [Val Loss] Tot: 0.4547 | MHR: 0.0101 | Struct: 0.0001 | 2D: 0.1652 | 3D: 0.1406\n",- " π [Val Metrics] Mesh PA-MPJPE: 149.9 mm | Mesh MPJPE: 205.1 mm\n",- " π [Val Metrics] Head PA-MPJPE: 158.9 mm | Head MPJPE: 250.1 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "03dc5a39d5e14b2794ddeda7fb776dc9",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 13/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "9ff89bf166ee421b9b2b51c2df979c9d",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 13/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 13 Summary | LR: 1.26e-04\n",- " [Train Loss] Tot: 0.3499 | MHR: 0.0099 | Struct: 0.0001 | 2D: 0.1179 | 3D: 0.1121\n",- " [Val Loss] Tot: 0.4111 | MHR: 0.0102 | Struct: 0.0001 | 2D: 0.1413 | 3D: 0.1254\n",- " π [Val Metrics] Mesh PA-MPJPE: 133.9 mm | Mesh MPJPE: 201.7 mm\n",- " π [Val Metrics] Head PA-MPJPE: 144.9 mm | Head MPJPE: 233.2 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 133.9 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "d390a8294f134275b4a19c01f74ee6bb",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 14/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "6bd67fcbad0e40ce9197dcb889bc6497",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 14/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 14 Summary | LR: 1.41e-04\n",- " [Train Loss] Tot: 0.3148 | MHR: 0.0095 | Struct: 0.0001 | 2D: 0.1027 | 3D: 0.0998\n",- " [Val Loss] Tot: 0.4096 | MHR: 0.0099 | Struct: 0.0001 | 2D: 0.1372 | 3D: 0.1200\n",- " π [Val Metrics] Mesh PA-MPJPE: 121.9 mm | Mesh MPJPE: 195.7 mm\n",- " π [Val Metrics] Head PA-MPJPE: 140.1 mm | Head MPJPE: 222.4 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 121.9 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "abae857407fb43edad80ac1b968f2a58",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 15/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "7e722eff257f430eba93de86a0392eff",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 15/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 15 Summary | LR: 1.56e-04\n",- " [Train Loss] Tot: 0.2847 | MHR: 0.0092 | Struct: 0.0001 | 2D: 0.0908 | 3D: 0.0894\n",- " [Val Loss] Tot: 0.3955 | MHR: 0.0097 | Struct: 0.0001 | 2D: 0.1401 | 3D: 0.1162\n",- " π [Val Metrics] Mesh PA-MPJPE: 117.1 mm | Mesh MPJPE: 196.8 mm\n",- " π [Val Metrics] Head PA-MPJPE: 135.3 mm | Head MPJPE: 218.4 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 117.1 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "16038f06363f442aa64da23ef17b5f39",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 16/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "51d0744b60974c23bb4f47741159c6ca",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 16/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 16 Summary | LR: 1.71e-04\n",- " [Train Loss] Tot: 0.2609 | MHR: 0.0088 | Struct: 0.0001 | 2D: 0.0813 | 3D: 0.0810\n",- " [Val Loss] Tot: 0.3872 | MHR: 0.0095 | Struct: 0.0001 | 2D: 0.1198 | 3D: 0.1081\n",- " π [Val Metrics] Mesh PA-MPJPE: 134.7 mm | Mesh MPJPE: 192.6 mm\n",- " π [Val Metrics] Head PA-MPJPE: 133.8 mm | Head MPJPE: 205.4 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "7528e8540e6e4ad79de90e36b89caf22",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 17/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "3414a7eef4b64bdc90e8c76a29be534c",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 17/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 17 Summary | LR: 1.86e-04\n",- " [Train Loss] Tot: 0.2446 | MHR: 0.0087 | Struct: 0.0001 | 2D: 0.0752 | 3D: 0.0757\n",- " [Val Loss] Tot: 0.3706 | MHR: 0.0093 | Struct: 0.0000 | 2D: 0.1198 | 3D: 0.1057\n",- " π [Val Metrics] Mesh PA-MPJPE: 117.3 mm | Mesh MPJPE: 183.4 mm\n",- " π [Val Metrics] Head PA-MPJPE: 128.6 mm | Head MPJPE: 203.9 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "a2ec4aa543814a98a9c1c3b95f6cd515",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 18/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "50a3c8dacba544c2aa55f923caa57706",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 18/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 18 Summary | LR: 2.00e-04\n",- " [Train Loss] Tot: 0.2312 | MHR: 0.0084 | Struct: 0.0001 | 2D: 0.0696 | 3D: 0.0714\n",- " [Val Loss] Tot: 0.3721 | MHR: 0.0095 | Struct: 0.0000 | 2D: 0.1298 | 3D: 0.1122\n",- " π [Val Metrics] Mesh PA-MPJPE: 135.7 mm | Mesh MPJPE: 194.6 mm\n",- " π [Val Metrics] Head PA-MPJPE: 137.1 mm | Head MPJPE: 216.3 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "510dd15a0c374bbd80e6a932ca267f68",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 19/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "ff1d4cd4ef014e1fb5600f461144a71c",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 19/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 19 Summary | LR: 2.14e-04\n",- " [Train Loss] Tot: 0.2188 | MHR: 0.0081 | Struct: 0.0001 | 2D: 0.0650 | 3D: 0.0666\n",- " [Val Loss] Tot: 0.3438 | MHR: 0.0091 | Struct: 0.0000 | 2D: 0.1143 | 3D: 0.1070\n",- " π [Val Metrics] Mesh PA-MPJPE: 107.3 mm | Mesh MPJPE: 189.8 mm\n",- " π [Val Metrics] Head PA-MPJPE: 124.7 mm | Head MPJPE: 201.3 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 107.3 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "05e8de05f6944861a66e928626dcff88",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 20/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "4a2b9ae1ab404af5b50708c83a27248c",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 20/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 20 Summary | LR: 2.28e-04\n",- " [Train Loss] Tot: 0.2026 | MHR: 0.0079 | Struct: 0.0001 | 2D: 0.0590 | 3D: 0.0612\n",- " [Val Loss] Tot: 0.3487 | MHR: 0.0089 | Struct: 0.0001 | 2D: 0.1193 | 3D: 0.0998\n",- " π [Val Metrics] Mesh PA-MPJPE: 114.4 mm | Mesh MPJPE: 183.5 mm\n",- " π [Val Metrics] Head PA-MPJPE: 125.6 mm | Head MPJPE: 201.0 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "bed7ceecef86415a80ea62d81dd1053e",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 21/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "c7159851d3b74de089416af481d77f8a",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 21/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 21 Summary | LR: 2.41e-04\n",- " [Train Loss] Tot: 0.1964 | MHR: 0.0078 | Struct: 0.0001 | 2D: 0.0568 | 3D: 0.0581\n",- " [Val Loss] Tot: 0.3375 | MHR: 0.0090 | Struct: 0.0000 | 2D: 0.1102 | 3D: 0.1024\n",- " π [Val Metrics] Mesh PA-MPJPE: 112.2 mm | Mesh MPJPE: 182.5 mm\n",- " π [Val Metrics] Head PA-MPJPE: 124.6 mm | Head MPJPE: 196.6 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "4da27b868dde4a33a75815a5a133bc05",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 22/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "62eb485afcf14d739d2f10fac592a114",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 22/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 22 Summary | LR: 2.52e-04\n",- " [Train Loss] Tot: 0.1913 | MHR: 0.0077 | Struct: 0.0001 | 2D: 0.0553 | 3D: 0.0568\n",- " [Val Loss] Tot: 0.3714 | MHR: 0.0087 | Struct: 0.0000 | 2D: 0.1194 | 3D: 0.1019\n",- " π [Val Metrics] Mesh PA-MPJPE: 112.2 mm | Mesh MPJPE: 179.2 mm\n",- " π [Val Metrics] Head PA-MPJPE: 126.4 mm | Head MPJPE: 200.7 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "b9419ba22b6d4bfc99481d4a7e3405d4",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 23/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "40a07bb539f04b5ba3a9efb4057e5889",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 23/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 23 Summary | LR: 2.63e-04\n",- " [Train Loss] Tot: 0.1909 | MHR: 0.0077 | Struct: 0.0001 | 2D: 0.0546 | 3D: 0.0569\n",- " [Val Loss] Tot: 0.3497 | MHR: 0.0089 | Struct: 0.0000 | 2D: 0.1063 | 3D: 0.0929\n",- " π [Val Metrics] Mesh PA-MPJPE: 114.5 mm | Mesh MPJPE: 175.1 mm\n",- " π [Val Metrics] Head PA-MPJPE: 124.6 mm | Head MPJPE: 193.2 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "064120901d504062a11868ce7ad3564f",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 24/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "10b69f6b02cb46d1a5a54e246b687be8",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 24/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 24 Summary | LR: 2.72e-04\n",- " [Train Loss] Tot: 0.1750 | MHR: 0.0075 | Struct: 0.0001 | 2D: 0.0489 | 3D: 0.0507\n",- " [Val Loss] Tot: 0.3207 | MHR: 0.0087 | Struct: 0.0000 | 2D: 0.1092 | 3D: 0.0920\n",- " π [Val Metrics] Mesh PA-MPJPE: 118.1 mm | Mesh MPJPE: 179.4 mm\n",- " π [Val Metrics] Head PA-MPJPE: 121.9 mm | Head MPJPE: 186.9 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "6c0828e4eb6d41c18c7f0762fed114cb",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 25/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "ca5221d7681f46989432d0233ebbbc02",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 25/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 25 Summary | LR: 2.81e-04\n",- " [Train Loss] Tot: 0.1783 | MHR: 0.0075 | Struct: 0.0001 | 2D: 0.0503 | 3D: 0.0521\n",- " [Val Loss] Tot: 0.3289 | MHR: 0.0090 | Struct: 0.0001 | 2D: 0.1070 | 3D: 0.0970\n",- " π [Val Metrics] Mesh PA-MPJPE: 108.6 mm | Mesh MPJPE: 180.0 mm\n",- " π [Val Metrics] Head PA-MPJPE: 120.4 mm | Head MPJPE: 195.1 mm\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "e3afdba61dc849fe9836700dfd28ca2a",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 26/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "2c286c2d0bf646cba6a994777475e111",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 26/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- },- {- "name": "stdout",- "output_type": "stream",- "text": [- "\n",- "π Epoch 26 Summary | LR: 2.87e-04\n",- " [Train Loss] Tot: 0.1769 | MHR: 0.0076 | Struct: 0.0001 | 2D: 0.0506 | 3D: 0.0507\n",- " [Val Loss] Tot: 0.3188 | MHR: 0.0088 | Struct: 0.0001 | 2D: 0.0991 | 3D: 0.0910\n",- " π [Val Metrics] Mesh PA-MPJPE: 103.9 mm | Mesh MPJPE: 194.1 mm\n",- " π [Val Metrics] Head PA-MPJPE: 116.1 mm | Head MPJPE: 189.8 mm\n",- "πΎ New Best Model saved! Mesh PA-MPJPE dropped to 103.9 mm.\n"- ]- },- {- "data": {- "application/vnd.jupyter.widget-view+json": {- "model_id": "834eb54a90bf4df1a945b644758aebc0",- "version_major": 2,- "version_minor": 0- },- "text/plain": [- "Epoch 27/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"- ]- },- "metadata": {},- "output_type": "display_data"- }- ],+ "outputs": [],"source": ["# ============================================================\n",- "# Cell 11 β The Full Distillation Training Loop (Phase 2/Fresh)\n",+ "# Cell 11 β Full Distillation Training Loop (EMA + Early-Stopping)\n","# ============================================================\n","import torch\n","import torch.optim as optim\n",@@ -2719,171 +1624,174 @@"import os\n","import math\n","from tqdm.notebook import tqdm\n",+ "from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn\n",+ "\n",+ "# EMA / early-stopping knobs (override in cfg if you want; sane defaults otherwise)\n",+ "EMA_DECAY = getattr(cfg, \"ema_decay\", 0.9998)\n",+ "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 30) # epochs w/o val improvement\n","\n","def train_instant_hmr():\n",- " print(f\"--- Starting Full Training Pipeline: {cfg.backbone} ---\")\n",+ " print(f\"--- Training {cfg.backbone} | EMA decay={EMA_DECAY} | early-stop patience={EARLY_STOP_PATIENCE} ---\")\n"," os.makedirs(cfg.log_dir, exist_ok=True)\n",- " \n",- " # --- CONFIGURATION ---\n",- " RESUME_FROM_CHECKPOINT = False # Set to False to train from scratch\n",- " \n",- " # 1. Initialize Model & Loss\n",+ "\n",+ " RESUME_FROM_CHECKPOINT = False\n",+ "\n",+ " # 1. Model + EMA shadow + loss\n"," model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",+ " # use_buffers=True so RepViT BatchNorm running stats are tracked by the EMA too.\n",+ " ema_model = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(EMA_DECAY), use_buffers=True)\n"," criterion = DistillationLoss(cfg, mhr_module)\n",- " \n",- " # 2. Optimizer & AMP Scaler\n",+ "\n",+ " # 2. Optimizer & AMP\n"," optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n"," scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",- " \n",- " # --- RESUME / FRESH START LOGIC ---\n",- " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model.pth\")\n",- " start_epoch = 0 \n",- " best_val_metric = float('inf') \n",- " \n",+ "\n",+ " best_raw_path = os.path.join(cfg.log_dir, \"best_student_model_raw.pth\")\n",+ " best_ema_path = os.path.join(cfg.log_dir, \"best_student_model_ema.pth\")\n",+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall; loaded by export/eval cells\n",+ "\n"," if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n"," print(f\"π Resuming from checkpoint: {best_ckpt_path}\")\n"," ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",- " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False) \n",+ " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False)\n"," else:\n"," print(\"π Starting training from scratch with fresh weights.\")\n","\n",- " # Disable torch.compile() to prevent the LLVM pthread_join crash \n",- " # and to stop BatchNorm running stats from corrupting during validation.\n",- " # print(\"β³ Compiling model with torch.compile()...\")\n",- " # model = torch.compile(model, mode=\"default\") \n",- "\n",- " # 3. Learning Rate Scheduler\n",+ " # 3. LR schedule\n"," steps_per_epoch = len(train_loader)\n"," scheduler = lr_scheduler.OneCycleLR(\n",- " optimizer, \n",- " max_lr=cfg.lr, \n",- " epochs=cfg.epochs, \n",- " steps_per_epoch=steps_per_epoch,\n",- " pct_start=0.1\n",+ " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",+ " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n"," )\n",- " \n",- " # --- MAIN EPOCH LOOP ---\n",- " for epoch in range(start_epoch, cfg.epochs):\n",+ "\n",+ " # --- validation helper: evaluate ANY model over val_loader (closes over criterion/loaders) ---\n",+ " @torch.no_grad()\n",+ " def run_validation(eval_model, tag, epoch):\n",+ " eval_model.eval()\n",+ " vloss = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",+ " hmr = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",+ " n = 0\n",+ " vbar = tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]\", leave=False)\n",+ " for batch in vbar:\n",+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",+ " preds_fp16 = eval_model(batch[\"image\"], batch[\"cliff_cond\"])\n",+ " preds = {k: v.float() for k, v in preds_fp16.items()}\n",+ " losses = criterion(preds, batch)\n",+ " if math.isnan(losses['total_loss'].item()) or math.isinf(losses['total_loss'].item()):\n",+ " continue\n",+ " vloss['total'] += losses['total_loss'].item()\n",+ " vloss['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",+ " vloss['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",+ " vloss['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",+ " vloss['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",+ " bm = evaluate_hmr_batch(preds, batch, cfg, mhr_module)\n",+ " for k in hmr:\n",+ " hmr[k] += bm.get(k, 0.0)\n",+ " n += 1\n",+ " # live tqdm read-out of THIS model's running PA-MPJPE (so you see raw vs ema as they run)\n",+ " vbar.set_postfix({'PA-MPJPE': f\"{hmr['Mesh_PA_MPJPE']/max(n,1):.1f}mm\"})\n",+ " vbar.close()\n",+ " vloss = {k: v / max(n, 1) for k, v in vloss.items()}\n",+ " hmr = {k: v / max(n, 1) for k, v in hmr.items()}\n",+ " return vloss, hmr, n\n",+ "\n",+ " best_raw_pa = float('inf')\n",+ " best_ema_pa = float('inf')\n",+ " best_overall_pa = float('inf')\n",+ " epochs_no_improve = 0\n",+ "\n",+ " for epoch in range(cfg.epochs):\n",+ " # ---------------- TRAIN ----------------\n"," model.train()\n",- " \n",- " train_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",- " valid_train_batches = 0\n",- " \n",+ " tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",+ " nb = 0\n"," pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Train]\")\n"," for batch in pbar:\n"," batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",- " \n"," optimizer.zero_grad(set_to_none=True)\n",- " \n"," with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n"," preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",- " \n"," preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n"," losses = criterion(preds_fp32, batch)\n"," loss = losses['total_loss']\n",- " \n"," if math.isnan(loss.item()) or math.isinf(loss.item()):\n"," print(\"β οΈ WARNING: NaN/Inf loss detected! Skipping step.\")\n"," continue\n",- " \n"," scaler.scale(loss).backward()\n"," scaler.unscale_(optimizer)\n"," torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)\n",- " \n",- " # π Only step the scheduler if the optimizer actually stepped\n"," scale_before = scaler.get_scale()\n"," scaler.step(optimizer)\n"," scaler.update()\n"," scale_after = scaler.get_scale()\n",- " \n",- " # If the scale didn't drop (meaning no NaNs were found in gradients), step the LR scheduler\n",- " if scale_before <= scale_after:\n",+ " if scale_before <= scale_after: # optimizer actually stepped (no inf grads)\n"," scheduler.step()\n",- " \n",- " # Accumulate metrics\n",- " train_metrics['total'] += loss.item()\n",- " train_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",- " train_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",- " train_metrics['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",- " train_metrics['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",- " train_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",- " train_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",- " valid_train_batches += 1\n",- " \n",- " pbar.set_postfix({\n",- " 'Tot': f\"{loss.item():.3f}\", \n",- " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\",\n",- " 'Str': f\"{losses.get('loss_structural', torch.tensor(0)).item():.3f}\"\n",- " })\n",- " \n",- " pbar.close() \n",- " avg_train = {k: v / max(valid_train_batches, 1) for k, v in train_metrics.items()}\n",- " \n",- " # --- VALIDATION LOOP ---\n",- " model.eval()\n",- " val_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",- " \n",- " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",- " valid_val_batches = 0\n",- " \n",- " with torch.no_grad():\n",- " for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val]\", leave=False):\n",- " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",- " \n",- " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",- " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",- " \n",- " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",- " losses = criterion(preds_fp32, batch)\n",- " loss = losses['total_loss']\n",- " \n",- " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",- " continue\n",- " \n",- " val_metrics['total'] += loss.item()\n",- " val_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",- " val_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",- " val_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",- " val_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",- " \n",- " batch_hmr_metrics = evaluate_hmr_batch(preds_fp32, batch, cfg, mhr_module)\n",- " for k in hmr_trackers.keys():\n",- " hmr_trackers[k] += batch_hmr_metrics.get(k, 0.0)\n",- " \n",- " valid_val_batches += 1\n",- " \n",- " avg_val = {k: v / max(valid_val_batches, 1) for k, v in val_metrics.items()}\n",- " avg_hmr = {k: v / max(valid_val_batches, 1) for k, v in hmr_trackers.items()}\n",- " \n",- " # --- DETAILED EPOCH SUMMARY ---\n",+ " ema_model.update_parameters(model) # EMA update follows the real param step\n",+ " tm['total'] += loss.item()\n",+ " tm['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",+ " tm['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",+ " tm['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",+ " tm['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",+ " tm['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",+ " tm['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",+ " nb += 1\n",+ " pbar.set_postfix({'Tot': f\"{loss.item():.3f}\",\n",+ " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\"})\n",+ " pbar.close()\n",+ " avg_train = {k: v / max(nb, 1) for k, v in tm.items()}\n",+ "\n",+ " # ---------------- VALIDATE: raw model AND EMA model ----------------\n",+ " raw_val, raw_hmr, n_raw = run_validation(model, \"raw\", epoch)\n",+ " ema_val, ema_hmr, n_ema = run_validation(ema_model, \"ema\", epoch)\n",+ "\n",+ " # ---------------- SUMMARY ----------------\n"," print(f\"\\nπ Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}\")\n",- " print(f\" [Train Loss] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | Struct: {avg_train['struct']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",- " print(f\" [Val Loss] Tot: {avg_val['total']:.4f} | MHR: {avg_val['mhr']:.4f} | Struct: {avg_val['struct']:.4f} | 2D: {avg_val['2d']:.4f} | 3D: {avg_val['3d']:.4f}\")\n",- " print(f\" π [Val Metrics] Mesh PA-MPJPE: {avg_hmr['Mesh_PA_MPJPE']:.1f} mm | Mesh MPJPE: {avg_hmr['Mesh_MPJPE']:.1f} mm\")\n",- " print(f\" π [Val Metrics] Head PA-MPJPE: {avg_hmr['Head_3D_PA_MPJPE']:.1f} mm | Head MPJPE: {avg_hmr['Head_3D_MPJPE']:.1f} mm\")\n",- " \n",- " # --- CHECKPOINTING (Now based on Mesh PA-MPJPE!) ---\n",- " current_metric = avg_hmr['Mesh_PA_MPJPE']\n",- " if current_metric < best_val_metric and valid_val_batches > 0:\n",- " best_val_metric = current_metric\n",- " save_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",- " torch.save({\n",- " 'epoch': epoch,\n",- " 'model_state_dict': model.state_dict(),\n",- " 'optimizer_state_dict': optimizer.state_dict(),\n",- " 'val_pa_mpjpe': best_val_metric,\n",- " }, save_path)\n",- " print(f\"πΎ New Best Model saved! Mesh PA-MPJPE dropped to {best_val_metric:.1f} mm.\")\n",- "\n",- " print(\"\\nπ Training Complete!\")\n",+ " print(f\" [Train] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",+ " print(f\" [Val RAW] Tot: {raw_val['total']:.4f} | Mesh PA-MPJPE: {raw_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {raw_hmr['Mesh_MPJPE']:.1f} | Head PA: {raw_hmr['Head_3D_PA_MPJPE']:.1f} mm\")\n",+ " print(f\" [Val EMA] Tot: {ema_val['total']:.4f} | Mesh PA-MPJPE: {ema_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {ema_hmr['Mesh_MPJPE']:.1f} | Head PA: {ema_hmr['Head_3D_PA_MPJPE']:.1f} mm\")\n",+ " delta = raw_hmr['Mesh_PA_MPJPE'] - ema_hmr['Mesh_PA_MPJPE']\n",+ " print(f\" π¬ EMA vs RAW (Mesh PA-MPJPE): {'EMA better' if delta > 0 else 'RAW better'} by {abs(delta):.1f} mm\")\n",+ "\n",+ " # ---------------- CHECKPOINTS (best of each, separately) ----------------\n",+ " if n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa:\n",+ " best_raw_pa = raw_hmr['Mesh_PA_MPJPE']\n",+ " torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(),\n",+ " 'val_pa_mpjpe': best_raw_pa, 'source': 'raw'}, best_raw_path)\n",+ " if n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa:\n",+ " best_ema_pa = ema_hmr['Mesh_PA_MPJPE']\n",+ " # ema_model.module is the underlying InstantHMRStudent -> clean state_dict keys\n",+ " torch.save({'epoch': epoch, 'model_state_dict': ema_model.module.state_dict(),\n",+ " 'val_pa_mpjpe': best_ema_pa, 'source': 'ema'}, best_ema_path)\n",+ "\n",+ " # ---------------- EARLY STOPPING (on the better of raw/EMA) ----------------\n",+ " current = min(raw_hmr['Mesh_PA_MPJPE'], ema_hmr['Mesh_PA_MPJPE'])\n",+ " if current < best_overall_pa - 1e-4:\n",+ " best_overall_pa = current\n",+ " epochs_no_improve = 0\n",+ " use_ema = ema_hmr['Mesh_PA_MPJPE'] <= raw_hmr['Mesh_PA_MPJPE']\n",+ " better = ema_model.module if use_ema else model\n",+ " torch.save({'epoch': epoch, 'model_state_dict': better.state_dict(),\n",+ " 'val_pa_mpjpe': best_overall_pa, 'source': 'ema' if use_ema else 'raw'},\n",+ " best_ckpt_path)\n",+ " print(f\"πΎ New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",+ " else:\n",+ " epochs_no_improve += 1\n",+ " print(f\" β³ No improvement {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best {best_overall_pa:.1f} mm)\")\n",+ " if epochs_no_improve >= EARLY_STOP_PATIENCE:\n",+ " print(f\"\\nπ Early stopping at epoch {epoch+1}: no val improvement in {EARLY_STOP_PATIENCE} epochs.\")\n",+ " break\n",+ "\n",+ " print(f\"\\nπ Training complete! Best overall Mesh PA-MPJPE: {best_overall_pa:.1f} mm\")\n",+ " print(f\" best RAW: {best_raw_pa:.1f} mm -> {best_raw_path}\")\n",+ " print(f\" best EMA: {best_ema_pa:.1f} mm -> {best_ema_path}\")\n","\n","# Kick off the training!\n","if __name__ == \"__main__\":\n"," import gc\n"," gc.collect()\n"," torch.cuda.empty_cache()\n",- " \n",- " train_instant_hmr()"+ " train_instant_hmr()\n"]},{