Skip to content
Python

feat: added exponential moving average on model weights for regularization

mohamdev885594f768b2
diff --git a/notebooks/distill_transformer_decoder.ipynb b/notebooks/distill_transformer_decoder.ipynb
index 2745eac..badd8fb 100644
--- a/notebooks/distill_transformer_decoder.ipynb
+++ b/notebooks/distill_transformer_decoder.ipynb
@@ -37,22 +37,10 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"id": "7e2ae437",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Project root: /home/madjel/Projects/gitpackages/sam3d-student-release\n",
- "Images dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/images\n",
- "Annotations dir: /home/madjel/Projects/gitpackages/sam3d-student-release/data/sam3d_gt_coco/annotations\n",
- "MHR model path: /home/madjel/Projects/gitpackages/sam3d-student-release/checkpoints/mhr_model.pt\n",
- "Config loaded. Backbone: repvit_m2_3. Ready for V3 Training.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
"# Cell 1 β€” Configuration\n",
@@ -112,6 +100,8 @@
" warmup_epochs: int = 3\n",
" grad_clip: float = 5.0\n",
" use_amp: bool = True \n",
+ " ema_decay: float = 0.9998 # EMA shadow-weights decay (Cell 11)\n",
+ " early_stop_patience: int = 30 # stop after N epochs w/o val PA-MPJPE improvement (Cell 11)\n",
"\n",
" # --- Loss weights (UPDATED for HMR Balance) ---\n",
" w_pose: float = 1.0\n",
@@ -1623,1095 +1613,10 @@
"execution_count": null,
"id": "d44aeb4a",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "--- Starting Full Training Pipeline: repvit_m2_3 ---\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "Unexpected keys (head.head.bn.num_batches_tracked, head.head.bn.bias, head.head.bn.running_mean, head.head.bn.running_var, head.head.bn.weight, head.head_dist.bn.num_batches_tracked, head.head_dist.bn.bias, head.head_dist.bn.running_mean, head.head_dist.bn.running_var, head.head_dist.bn.weight) found while loading pretrained weights. This may be expected if model is being adapted.\n"
- ]
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "πŸš€ Starting training from scratch with fresh weights.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "87ac1c75193b444f90e0f56c736cb2c7",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 1/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "803af2b02f5e4ceda96daee5032a7cf9",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 1/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 1 Summary | LR: 1.28e-05\n",
- " [Train Loss] Tot: 1.7388 | MHR: 0.0153 | Struct: 0.0003 | 2D: 0.7564 | 3D: 0.6079\n",
- " [Val Loss] Tot: 1.5152 | MHR: 0.0149 | Struct: 0.0002 | 2D: 0.7518 | 3D: 0.4873\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 179.5 mm | Mesh MPJPE: 268.8 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 305.6 mm | Head MPJPE: 604.1 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 179.5 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "3020c4fe8796467ebc911f875a2b947e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 2/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "19f6af9412c6427c8a8d6915ebedf21f",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 2/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 2 Summary | LR: 1.51e-05\n",
- " [Train Loss] Tot: 1.4542 | MHR: 0.0149 | Struct: 0.0002 | 2D: 0.7299 | 3D: 0.4811\n",
- " [Val Loss] Tot: 1.4444 | MHR: 0.0148 | Struct: 0.0001 | 2D: 0.7298 | 3D: 0.4807\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 188.0 mm | Mesh MPJPE: 265.7 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 273.5 mm | Head MPJPE: 602.0 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "fe0d1a4cabba48eab7ef2c4a60371c3b",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 3/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "28c4e59d28444e9aa3e6dad69996539d",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 3/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 3 Summary | LR: 1.90e-05\n",
- " [Train Loss] Tot: 1.2657 | MHR: 0.0148 | Struct: 0.0001 | 2D: 0.6214 | 3D: 0.4171\n",
- " [Val Loss] Tot: 1.0030 | MHR: 0.0149 | Struct: 0.0001 | 2D: 0.4181 | 3D: 0.3143\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 164.4 mm | Mesh MPJPE: 279.0 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 268.2 mm | Head MPJPE: 361.1 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 164.4 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "3e1f88f6fd344f48b7cc1d2c7f2f2e11",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 4/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "7e99577ffdf942e48c581b14cafcdb6b",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 4/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 4 Summary | LR: 2.44e-05\n",
- " [Train Loss] Tot: 0.9220 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.4073 | 3D: 0.3005\n",
- " [Val Loss] Tot: 0.9751 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.4150 | 3D: 0.2991\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 188.7 mm | Mesh MPJPE: 266.9 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 265.9 mm | Head MPJPE: 350.1 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "5e00a1d07f274859af01b22b16f17dd9",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 5/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "f585f7a075344ffc94155af8c0db1d77",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 5/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 5 Summary | LR: 3.12e-05\n",
- " [Train Loss] Tot: 0.8757 | MHR: 0.0147 | Struct: 0.0001 | 2D: 0.3762 | 3D: 0.2941\n",
- " [Val Loss] Tot: 0.8734 | MHR: 0.0145 | Struct: 0.0001 | 2D: 0.3690 | 3D: 0.2949\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 167.5 mm | Mesh MPJPE: 273.3 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 244.7 mm | Head MPJPE: 345.8 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "dc32d42921644e9481bc743c538c7486",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 6/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ac683d5f8b7043248b6cbaace330fdc5",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 6/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 6 Summary | LR: 3.94e-05\n",
- " [Train Loss] Tot: 0.8386 | MHR: 0.0145 | Struct: 0.0001 | 2D: 0.3555 | 3D: 0.2861\n",
- " [Val Loss] Tot: 0.8797 | MHR: 0.0144 | Struct: 0.0001 | 2D: 0.3649 | 3D: 0.2821\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 174.5 mm | Mesh MPJPE: 269.4 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 236.2 mm | Head MPJPE: 337.8 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d3e192c380ac40bd9893dc54989cd764",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 7/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "0c8f9fef643448c3b7d28719335d4726",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 7/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 7 Summary | LR: 4.89e-05\n",
- " [Train Loss] Tot: 0.8004 | MHR: 0.0144 | Struct: 0.0001 | 2D: 0.3354 | 3D: 0.2764\n",
- " [Val Loss] Tot: 0.8025 | MHR: 0.0137 | Struct: 0.0000 | 2D: 0.3330 | 3D: 0.2679\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 174.4 mm | Mesh MPJPE: 259.5 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 219.4 mm | Head MPJPE: 330.2 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "a821d048197346e599af1307b601d1ed",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 8/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "986d9d359f7f43c590976f964bba2fc9",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 8/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 8 Summary | LR: 5.95e-05\n",
- " [Train Loss] Tot: 0.7231 | MHR: 0.0136 | Struct: 0.0001 | 2D: 0.2984 | 3D: 0.2468\n",
- " [Val Loss] Tot: 4.0354 | MHR: 0.0138 | Struct: 0.0002 | 2D: 0.3756 | 3D: 0.3284\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 155.2 mm | Mesh MPJPE: 258.9 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 215.3 mm | Head MPJPE: 326.2 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 155.2 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d5b710c460904c0ca501d007102a5e88",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 9/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "0bff56f501b945f89aed52c07a4799c4",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 9/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 9 Summary | LR: 7.13e-05\n",
- " [Train Loss] Tot: 0.6247 | MHR: 0.0126 | Struct: 0.0001 | 2D: 0.2521 | 3D: 0.2064\n",
- " [Val Loss] Tot: 0.6047 | MHR: 0.0122 | Struct: 0.0001 | 2D: 0.2531 | 3D: 0.1814\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 171.1 mm | Mesh MPJPE: 232.1 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 193.6 mm | Head MPJPE: 290.4 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "2421f23dd24e4a97b67ec7e52b9fbb7b",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 10/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "5f62154c7c3b44518cd53afc2fe56253",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 10/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 10 Summary | LR: 8.39e-05\n",
- " [Train Loss] Tot: 0.5327 | MHR: 0.0117 | Struct: 0.0001 | 2D: 0.2063 | 3D: 0.1734\n",
- " [Val Loss] Tot: 0.5420 | MHR: 0.0113 | Struct: 0.0001 | 2D: 0.1963 | 3D: 0.1619\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 134.2 mm | Mesh MPJPE: 227.0 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 169.2 mm | Head MPJPE: 270.3 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 134.2 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "e7f7207d3ae84a0582c9550e50aab0d8",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 11/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "26284798653b49c2b72f127be1570e5d",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 11/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 11 Summary | LR: 9.73e-05\n",
- " [Train Loss] Tot: 0.4547 | MHR: 0.0109 | Struct: 0.0001 | 2D: 0.1685 | 3D: 0.1470\n",
- " [Val Loss] Tot: 0.4793 | MHR: 0.0104 | Struct: 0.0001 | 2D: 0.1884 | 3D: 0.1442\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 145.8 mm | Mesh MPJPE: 210.3 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 159.2 mm | Head MPJPE: 255.8 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "7b2f8d8116544c66af0d55a98c45baaa",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 12/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "9928b32d7dea4e019765c3698cec5304",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 12/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 12 Summary | LR: 1.11e-04\n",
- " [Train Loss] Tot: 0.3963 | MHR: 0.0104 | Struct: 0.0001 | 2D: 0.1397 | 3D: 0.1282\n",
- " [Val Loss] Tot: 0.4547 | MHR: 0.0101 | Struct: 0.0001 | 2D: 0.1652 | 3D: 0.1406\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 149.9 mm | Mesh MPJPE: 205.1 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 158.9 mm | Head MPJPE: 250.1 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "03dc5a39d5e14b2794ddeda7fb776dc9",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 13/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "9ff89bf166ee421b9b2b51c2df979c9d",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 13/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 13 Summary | LR: 1.26e-04\n",
- " [Train Loss] Tot: 0.3499 | MHR: 0.0099 | Struct: 0.0001 | 2D: 0.1179 | 3D: 0.1121\n",
- " [Val Loss] Tot: 0.4111 | MHR: 0.0102 | Struct: 0.0001 | 2D: 0.1413 | 3D: 0.1254\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 133.9 mm | Mesh MPJPE: 201.7 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 144.9 mm | Head MPJPE: 233.2 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 133.9 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "d390a8294f134275b4a19c01f74ee6bb",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 14/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "6bd67fcbad0e40ce9197dcb889bc6497",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 14/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 14 Summary | LR: 1.41e-04\n",
- " [Train Loss] Tot: 0.3148 | MHR: 0.0095 | Struct: 0.0001 | 2D: 0.1027 | 3D: 0.0998\n",
- " [Val Loss] Tot: 0.4096 | MHR: 0.0099 | Struct: 0.0001 | 2D: 0.1372 | 3D: 0.1200\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 121.9 mm | Mesh MPJPE: 195.7 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 140.1 mm | Head MPJPE: 222.4 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 121.9 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "abae857407fb43edad80ac1b968f2a58",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 15/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "7e722eff257f430eba93de86a0392eff",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 15/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 15 Summary | LR: 1.56e-04\n",
- " [Train Loss] Tot: 0.2847 | MHR: 0.0092 | Struct: 0.0001 | 2D: 0.0908 | 3D: 0.0894\n",
- " [Val Loss] Tot: 0.3955 | MHR: 0.0097 | Struct: 0.0001 | 2D: 0.1401 | 3D: 0.1162\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 117.1 mm | Mesh MPJPE: 196.8 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 135.3 mm | Head MPJPE: 218.4 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 117.1 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "16038f06363f442aa64da23ef17b5f39",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 16/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "51d0744b60974c23bb4f47741159c6ca",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 16/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 16 Summary | LR: 1.71e-04\n",
- " [Train Loss] Tot: 0.2609 | MHR: 0.0088 | Struct: 0.0001 | 2D: 0.0813 | 3D: 0.0810\n",
- " [Val Loss] Tot: 0.3872 | MHR: 0.0095 | Struct: 0.0001 | 2D: 0.1198 | 3D: 0.1081\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 134.7 mm | Mesh MPJPE: 192.6 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 133.8 mm | Head MPJPE: 205.4 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "7528e8540e6e4ad79de90e36b89caf22",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 17/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "3414a7eef4b64bdc90e8c76a29be534c",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 17/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 17 Summary | LR: 1.86e-04\n",
- " [Train Loss] Tot: 0.2446 | MHR: 0.0087 | Struct: 0.0001 | 2D: 0.0752 | 3D: 0.0757\n",
- " [Val Loss] Tot: 0.3706 | MHR: 0.0093 | Struct: 0.0000 | 2D: 0.1198 | 3D: 0.1057\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 117.3 mm | Mesh MPJPE: 183.4 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 128.6 mm | Head MPJPE: 203.9 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "a2ec4aa543814a98a9c1c3b95f6cd515",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 18/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "50a3c8dacba544c2aa55f923caa57706",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 18/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 18 Summary | LR: 2.00e-04\n",
- " [Train Loss] Tot: 0.2312 | MHR: 0.0084 | Struct: 0.0001 | 2D: 0.0696 | 3D: 0.0714\n",
- " [Val Loss] Tot: 0.3721 | MHR: 0.0095 | Struct: 0.0000 | 2D: 0.1298 | 3D: 0.1122\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 135.7 mm | Mesh MPJPE: 194.6 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 137.1 mm | Head MPJPE: 216.3 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "510dd15a0c374bbd80e6a932ca267f68",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 19/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ff1d4cd4ef014e1fb5600f461144a71c",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 19/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 19 Summary | LR: 2.14e-04\n",
- " [Train Loss] Tot: 0.2188 | MHR: 0.0081 | Struct: 0.0001 | 2D: 0.0650 | 3D: 0.0666\n",
- " [Val Loss] Tot: 0.3438 | MHR: 0.0091 | Struct: 0.0000 | 2D: 0.1143 | 3D: 0.1070\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 107.3 mm | Mesh MPJPE: 189.8 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 124.7 mm | Head MPJPE: 201.3 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 107.3 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "05e8de05f6944861a66e928626dcff88",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 20/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "4a2b9ae1ab404af5b50708c83a27248c",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 20/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 20 Summary | LR: 2.28e-04\n",
- " [Train Loss] Tot: 0.2026 | MHR: 0.0079 | Struct: 0.0001 | 2D: 0.0590 | 3D: 0.0612\n",
- " [Val Loss] Tot: 0.3487 | MHR: 0.0089 | Struct: 0.0001 | 2D: 0.1193 | 3D: 0.0998\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 114.4 mm | Mesh MPJPE: 183.5 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 125.6 mm | Head MPJPE: 201.0 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "bed7ceecef86415a80ea62d81dd1053e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 21/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "c7159851d3b74de089416af481d77f8a",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 21/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 21 Summary | LR: 2.41e-04\n",
- " [Train Loss] Tot: 0.1964 | MHR: 0.0078 | Struct: 0.0001 | 2D: 0.0568 | 3D: 0.0581\n",
- " [Val Loss] Tot: 0.3375 | MHR: 0.0090 | Struct: 0.0000 | 2D: 0.1102 | 3D: 0.1024\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 112.2 mm | Mesh MPJPE: 182.5 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 124.6 mm | Head MPJPE: 196.6 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "4da27b868dde4a33a75815a5a133bc05",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 22/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "62eb485afcf14d739d2f10fac592a114",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 22/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 22 Summary | LR: 2.52e-04\n",
- " [Train Loss] Tot: 0.1913 | MHR: 0.0077 | Struct: 0.0001 | 2D: 0.0553 | 3D: 0.0568\n",
- " [Val Loss] Tot: 0.3714 | MHR: 0.0087 | Struct: 0.0000 | 2D: 0.1194 | 3D: 0.1019\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 112.2 mm | Mesh MPJPE: 179.2 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 126.4 mm | Head MPJPE: 200.7 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "b9419ba22b6d4bfc99481d4a7e3405d4",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 23/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "40a07bb539f04b5ba3a9efb4057e5889",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 23/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 23 Summary | LR: 2.63e-04\n",
- " [Train Loss] Tot: 0.1909 | MHR: 0.0077 | Struct: 0.0001 | 2D: 0.0546 | 3D: 0.0569\n",
- " [Val Loss] Tot: 0.3497 | MHR: 0.0089 | Struct: 0.0000 | 2D: 0.1063 | 3D: 0.0929\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 114.5 mm | Mesh MPJPE: 175.1 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 124.6 mm | Head MPJPE: 193.2 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "064120901d504062a11868ce7ad3564f",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 24/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "10b69f6b02cb46d1a5a54e246b687be8",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 24/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 24 Summary | LR: 2.72e-04\n",
- " [Train Loss] Tot: 0.1750 | MHR: 0.0075 | Struct: 0.0001 | 2D: 0.0489 | 3D: 0.0507\n",
- " [Val Loss] Tot: 0.3207 | MHR: 0.0087 | Struct: 0.0000 | 2D: 0.1092 | 3D: 0.0920\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 118.1 mm | Mesh MPJPE: 179.4 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 121.9 mm | Head MPJPE: 186.9 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "6c0828e4eb6d41c18c7f0762fed114cb",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 25/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ca5221d7681f46989432d0233ebbbc02",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 25/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 25 Summary | LR: 2.81e-04\n",
- " [Train Loss] Tot: 0.1783 | MHR: 0.0075 | Struct: 0.0001 | 2D: 0.0503 | 3D: 0.0521\n",
- " [Val Loss] Tot: 0.3289 | MHR: 0.0090 | Struct: 0.0001 | 2D: 0.1070 | 3D: 0.0970\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 108.6 mm | Mesh MPJPE: 180.0 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 120.4 mm | Head MPJPE: 195.1 mm\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "e3afdba61dc849fe9836700dfd28ca2a",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 26/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "2c286c2d0bf646cba6a994777475e111",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 26/300 [Val]: 0%| | 0/62 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "πŸ“ˆ Epoch 26 Summary | LR: 2.87e-04\n",
- " [Train Loss] Tot: 0.1769 | MHR: 0.0076 | Struct: 0.0001 | 2D: 0.0506 | 3D: 0.0507\n",
- " [Val Loss] Tot: 0.3188 | MHR: 0.0088 | Struct: 0.0001 | 2D: 0.0991 | 3D: 0.0910\n",
- " πŸ“ [Val Metrics] Mesh PA-MPJPE: 103.9 mm | Mesh MPJPE: 194.1 mm\n",
- " πŸ“ [Val Metrics] Head PA-MPJPE: 116.1 mm | Head MPJPE: 189.8 mm\n",
- "πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to 103.9 mm.\n"
- ]
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "834eb54a90bf4df1a945b644758aebc0",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Epoch 27/300 [Train]: 0%| | 0/557 [00:00<?, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
+ "outputs": [],
"source": [
"# ============================================================\n",
- "# Cell 11 β€” The Full Distillation Training Loop (Phase 2/Fresh)\n",
+ "# Cell 11 β€” Full Distillation Training Loop (EMA + Early-Stopping)\n",
"# ============================================================\n",
"import torch\n",
"import torch.optim as optim\n",
@@ -2719,171 +1624,174 @@
"import os\n",
"import math\n",
"from tqdm.notebook import tqdm\n",
+ "from torch.optim.swa_utils import AveragedModel, get_ema_multi_avg_fn\n",
+ "\n",
+ "# EMA / early-stopping knobs (override in cfg if you want; sane defaults otherwise)\n",
+ "EMA_DECAY = getattr(cfg, \"ema_decay\", 0.9998)\n",
+ "EARLY_STOP_PATIENCE = getattr(cfg, \"early_stop_patience\", 30) # epochs w/o val improvement\n",
"\n",
"def train_instant_hmr():\n",
- " print(f\"--- Starting Full Training Pipeline: {cfg.backbone} ---\")\n",
+ " print(f\"--- Training {cfg.backbone} | EMA decay={EMA_DECAY} | early-stop patience={EARLY_STOP_PATIENCE} ---\")\n",
" os.makedirs(cfg.log_dir, exist_ok=True)\n",
- " \n",
- " # --- CONFIGURATION ---\n",
- " RESUME_FROM_CHECKPOINT = False # Set to False to train from scratch\n",
- " \n",
- " # 1. Initialize Model & Loss\n",
+ "\n",
+ " RESUME_FROM_CHECKPOINT = False\n",
+ "\n",
+ " # 1. Model + EMA shadow + loss\n",
" model = InstantHMRStudent(cfg, pretrained=True).to(device)\n",
+ " # use_buffers=True so RepViT BatchNorm running stats are tracked by the EMA too.\n",
+ " ema_model = AveragedModel(model, multi_avg_fn=get_ema_multi_avg_fn(EMA_DECAY), use_buffers=True)\n",
" criterion = DistillationLoss(cfg, mhr_module)\n",
- " \n",
- " # 2. Optimizer & AMP Scaler\n",
+ "\n",
+ " # 2. Optimizer & AMP\n",
" optimizer = optim.AdamW(model.parameters(), lr=cfg.lr, weight_decay=cfg.weight_decay)\n",
" scaler = torch.amp.GradScaler(device='cuda', enabled=cfg.use_amp)\n",
- " \n",
- " # --- RESUME / FRESH START LOGIC ---\n",
- " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model.pth\")\n",
- " start_epoch = 0 \n",
- " best_val_metric = float('inf') \n",
- " \n",
+ "\n",
+ " best_raw_path = os.path.join(cfg.log_dir, \"best_student_model_raw.pth\")\n",
+ " best_ema_path = os.path.join(cfg.log_dir, \"best_student_model_ema.pth\")\n",
+ " best_ckpt_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\") # best overall; loaded by export/eval cells\n",
+ "\n",
" if RESUME_FROM_CHECKPOINT and os.path.exists(best_ckpt_path):\n",
" print(f\"πŸ”„ Resuming from checkpoint: {best_ckpt_path}\")\n",
" ckpt = torch.load(best_ckpt_path, map_location=device, weights_only=True)\n",
- " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False) \n",
+ " model.load_state_dict(ckpt[\"model_state_dict\"], strict=False)\n",
" else:\n",
" print(\"πŸš€ Starting training from scratch with fresh weights.\")\n",
"\n",
- " # Disable torch.compile() to prevent the LLVM pthread_join crash \n",
- " # and to stop BatchNorm running stats from corrupting during validation.\n",
- " # print(\"⏳ Compiling model with torch.compile()...\")\n",
- " # model = torch.compile(model, mode=\"default\") \n",
- "\n",
- " # 3. Learning Rate Scheduler\n",
+ " # 3. LR schedule\n",
" steps_per_epoch = len(train_loader)\n",
" scheduler = lr_scheduler.OneCycleLR(\n",
- " optimizer, \n",
- " max_lr=cfg.lr, \n",
- " epochs=cfg.epochs, \n",
- " steps_per_epoch=steps_per_epoch,\n",
- " pct_start=0.1\n",
+ " optimizer, max_lr=cfg.lr, epochs=cfg.epochs,\n",
+ " steps_per_epoch=steps_per_epoch, pct_start=0.1,\n",
" )\n",
- " \n",
- " # --- MAIN EPOCH LOOP ---\n",
- " for epoch in range(start_epoch, cfg.epochs):\n",
+ "\n",
+ " # --- validation helper: evaluate ANY model over val_loader (closes over criterion/loaders) ---\n",
+ " @torch.no_grad()\n",
+ " def run_validation(eval_model, tag, epoch):\n",
+ " eval_model.eval()\n",
+ " vloss = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
+ " hmr = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",
+ " n = 0\n",
+ " vbar = tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val:{tag}]\", leave=False)\n",
+ " for batch in vbar:\n",
+ " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
+ " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
+ " preds_fp16 = eval_model(batch[\"image\"], batch[\"cliff_cond\"])\n",
+ " preds = {k: v.float() for k, v in preds_fp16.items()}\n",
+ " losses = criterion(preds, batch)\n",
+ " if math.isnan(losses['total_loss'].item()) or math.isinf(losses['total_loss'].item()):\n",
+ " continue\n",
+ " vloss['total'] += losses['total_loss'].item()\n",
+ " vloss['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
+ " vloss['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
+ " vloss['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
+ " vloss['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
+ " bm = evaluate_hmr_batch(preds, batch, cfg, mhr_module)\n",
+ " for k in hmr:\n",
+ " hmr[k] += bm.get(k, 0.0)\n",
+ " n += 1\n",
+ " # live tqdm read-out of THIS model's running PA-MPJPE (so you see raw vs ema as they run)\n",
+ " vbar.set_postfix({'PA-MPJPE': f\"{hmr['Mesh_PA_MPJPE']/max(n,1):.1f}mm\"})\n",
+ " vbar.close()\n",
+ " vloss = {k: v / max(n, 1) for k, v in vloss.items()}\n",
+ " hmr = {k: v / max(n, 1) for k, v in hmr.items()}\n",
+ " return vloss, hmr, n\n",
+ "\n",
+ " best_raw_pa = float('inf')\n",
+ " best_ema_pa = float('inf')\n",
+ " best_overall_pa = float('inf')\n",
+ " epochs_no_improve = 0\n",
+ "\n",
+ " for epoch in range(cfg.epochs):\n",
+ " # ---------------- TRAIN ----------------\n",
" model.train()\n",
- " \n",
- " train_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
- " valid_train_batches = 0\n",
- " \n",
+ " tm = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'pose': 0.0, 'scale': 0.0, 'shape': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
+ " nb = 0\n",
" pbar = tqdm(train_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Train]\")\n",
" for batch in pbar:\n",
" batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
- " \n",
" optimizer.zero_grad(set_to_none=True)\n",
- " \n",
" with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
" preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",
- " \n",
" preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",
" losses = criterion(preds_fp32, batch)\n",
" loss = losses['total_loss']\n",
- " \n",
" if math.isnan(loss.item()) or math.isinf(loss.item()):\n",
" print(\"⚠️ WARNING: NaN/Inf loss detected! Skipping step.\")\n",
" continue\n",
- " \n",
" scaler.scale(loss).backward()\n",
" scaler.unscale_(optimizer)\n",
" torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)\n",
- " \n",
- " # πŸ›‘ Only step the scheduler if the optimizer actually stepped\n",
" scale_before = scaler.get_scale()\n",
" scaler.step(optimizer)\n",
" scaler.update()\n",
" scale_after = scaler.get_scale()\n",
- " \n",
- " # If the scale didn't drop (meaning no NaNs were found in gradients), step the LR scheduler\n",
- " if scale_before <= scale_after:\n",
+ " if scale_before <= scale_after: # optimizer actually stepped (no inf grads)\n",
" scheduler.step()\n",
- " \n",
- " # Accumulate metrics\n",
- " train_metrics['total'] += loss.item()\n",
- " train_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
- " train_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
- " train_metrics['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",
- " train_metrics['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",
- " train_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
- " train_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
- " valid_train_batches += 1\n",
- " \n",
- " pbar.set_postfix({\n",
- " 'Tot': f\"{loss.item():.3f}\", \n",
- " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\",\n",
- " 'Str': f\"{losses.get('loss_structural', torch.tensor(0)).item():.3f}\"\n",
- " })\n",
- " \n",
- " pbar.close() \n",
- " avg_train = {k: v / max(valid_train_batches, 1) for k, v in train_metrics.items()}\n",
- " \n",
- " # --- VALIDATION LOOP ---\n",
- " model.eval()\n",
- " val_metrics = {'total': 0.0, '2d': 0.0, '3d': 0.0, 'mhr': 0.0, 'struct': 0.0}\n",
- " \n",
- " hmr_trackers = {'Head_3D_MPJPE': 0.0, 'Head_3D_PA_MPJPE': 0.0, 'Mesh_MPJPE': 0.0, 'Mesh_PA_MPJPE': 0.0}\n",
- " valid_val_batches = 0\n",
- " \n",
- " with torch.no_grad():\n",
- " for batch in tqdm(val_loader, desc=f\"Epoch {epoch+1}/{cfg.epochs} [Val]\", leave=False):\n",
- " batch = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in batch.items()}\n",
- " \n",
- " with torch.amp.autocast(device_type='cuda', dtype=torch.float16, enabled=cfg.use_amp):\n",
- " preds_fp16 = model(batch[\"image\"], batch[\"cliff_cond\"])\n",
- " \n",
- " preds_fp32 = {k: v.float() for k, v in preds_fp16.items()}\n",
- " losses = criterion(preds_fp32, batch)\n",
- " loss = losses['total_loss']\n",
- " \n",
- " if math.isnan(loss.item()) or math.isinf(loss.item()):\n",
- " continue\n",
- " \n",
- " val_metrics['total'] += loss.item()\n",
- " val_metrics['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
- " val_metrics['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
- " val_metrics['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
- " val_metrics['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
- " \n",
- " batch_hmr_metrics = evaluate_hmr_batch(preds_fp32, batch, cfg, mhr_module)\n",
- " for k in hmr_trackers.keys():\n",
- " hmr_trackers[k] += batch_hmr_metrics.get(k, 0.0)\n",
- " \n",
- " valid_val_batches += 1\n",
- " \n",
- " avg_val = {k: v / max(valid_val_batches, 1) for k, v in val_metrics.items()}\n",
- " avg_hmr = {k: v / max(valid_val_batches, 1) for k, v in hmr_trackers.items()}\n",
- " \n",
- " # --- DETAILED EPOCH SUMMARY ---\n",
+ " ema_model.update_parameters(model) # EMA update follows the real param step\n",
+ " tm['total'] += loss.item()\n",
+ " tm['2d'] += losses.get('loss_2d_native', torch.tensor(0)).item()\n",
+ " tm['3d'] += losses.get('loss_3d_native', torch.tensor(0)).item()\n",
+ " tm['pose'] += losses.get('loss_pose', torch.tensor(0)).item()\n",
+ " tm['scale'] += losses.get('loss_scale', torch.tensor(0)).item()\n",
+ " tm['mhr'] += losses.get('loss_mhr_joints', torch.tensor(0)).item()\n",
+ " tm['struct'] += losses.get('loss_structural', torch.tensor(0)).item()\n",
+ " nb += 1\n",
+ " pbar.set_postfix({'Tot': f\"{loss.item():.3f}\",\n",
+ " 'MHR': f\"{losses.get('loss_mhr_joints', torch.tensor(0)).item():.3f}\"})\n",
+ " pbar.close()\n",
+ " avg_train = {k: v / max(nb, 1) for k, v in tm.items()}\n",
+ "\n",
+ " # ---------------- VALIDATE: raw model AND EMA model ----------------\n",
+ " raw_val, raw_hmr, n_raw = run_validation(model, \"raw\", epoch)\n",
+ " ema_val, ema_hmr, n_ema = run_validation(ema_model, \"ema\", epoch)\n",
+ "\n",
+ " # ---------------- SUMMARY ----------------\n",
" print(f\"\\nπŸ“ˆ Epoch {epoch+1} Summary | LR: {scheduler.get_last_lr()[0]:.2e}\")\n",
- " print(f\" [Train Loss] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | Struct: {avg_train['struct']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",
- " print(f\" [Val Loss] Tot: {avg_val['total']:.4f} | MHR: {avg_val['mhr']:.4f} | Struct: {avg_val['struct']:.4f} | 2D: {avg_val['2d']:.4f} | 3D: {avg_val['3d']:.4f}\")\n",
- " print(f\" πŸ“ [Val Metrics] Mesh PA-MPJPE: {avg_hmr['Mesh_PA_MPJPE']:.1f} mm | Mesh MPJPE: {avg_hmr['Mesh_MPJPE']:.1f} mm\")\n",
- " print(f\" πŸ“ [Val Metrics] Head PA-MPJPE: {avg_hmr['Head_3D_PA_MPJPE']:.1f} mm | Head MPJPE: {avg_hmr['Head_3D_MPJPE']:.1f} mm\")\n",
- " \n",
- " # --- CHECKPOINTING (Now based on Mesh PA-MPJPE!) ---\n",
- " current_metric = avg_hmr['Mesh_PA_MPJPE']\n",
- " if current_metric < best_val_metric and valid_val_batches > 0:\n",
- " best_val_metric = current_metric\n",
- " save_path = os.path.join(cfg.log_dir, \"best_student_model_v3.pth\")\n",
- " torch.save({\n",
- " 'epoch': epoch,\n",
- " 'model_state_dict': model.state_dict(),\n",
- " 'optimizer_state_dict': optimizer.state_dict(),\n",
- " 'val_pa_mpjpe': best_val_metric,\n",
- " }, save_path)\n",
- " print(f\"πŸ’Ύ New Best Model saved! Mesh PA-MPJPE dropped to {best_val_metric:.1f} mm.\")\n",
- "\n",
- " print(\"\\nπŸŽ‰ Training Complete!\")\n",
+ " print(f\" [Train] Tot: {avg_train['total']:.4f} | MHR: {avg_train['mhr']:.4f} | 2D: {avg_train['2d']:.4f} | 3D: {avg_train['3d']:.4f}\")\n",
+ " print(f\" [Val RAW] Tot: {raw_val['total']:.4f} | Mesh PA-MPJPE: {raw_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {raw_hmr['Mesh_MPJPE']:.1f} | Head PA: {raw_hmr['Head_3D_PA_MPJPE']:.1f} mm\")\n",
+ " print(f\" [Val EMA] Tot: {ema_val['total']:.4f} | Mesh PA-MPJPE: {ema_hmr['Mesh_PA_MPJPE']:.1f} | Mesh MPJPE: {ema_hmr['Mesh_MPJPE']:.1f} | Head PA: {ema_hmr['Head_3D_PA_MPJPE']:.1f} mm\")\n",
+ " delta = raw_hmr['Mesh_PA_MPJPE'] - ema_hmr['Mesh_PA_MPJPE']\n",
+ " print(f\" πŸ”¬ EMA vs RAW (Mesh PA-MPJPE): {'EMA better' if delta > 0 else 'RAW better'} by {abs(delta):.1f} mm\")\n",
+ "\n",
+ " # ---------------- CHECKPOINTS (best of each, separately) ----------------\n",
+ " if n_raw > 0 and raw_hmr['Mesh_PA_MPJPE'] < best_raw_pa:\n",
+ " best_raw_pa = raw_hmr['Mesh_PA_MPJPE']\n",
+ " torch.save({'epoch': epoch, 'model_state_dict': model.state_dict(),\n",
+ " 'val_pa_mpjpe': best_raw_pa, 'source': 'raw'}, best_raw_path)\n",
+ " if n_ema > 0 and ema_hmr['Mesh_PA_MPJPE'] < best_ema_pa:\n",
+ " best_ema_pa = ema_hmr['Mesh_PA_MPJPE']\n",
+ " # ema_model.module is the underlying InstantHMRStudent -> clean state_dict keys\n",
+ " torch.save({'epoch': epoch, 'model_state_dict': ema_model.module.state_dict(),\n",
+ " 'val_pa_mpjpe': best_ema_pa, 'source': 'ema'}, best_ema_path)\n",
+ "\n",
+ " # ---------------- EARLY STOPPING (on the better of raw/EMA) ----------------\n",
+ " current = min(raw_hmr['Mesh_PA_MPJPE'], ema_hmr['Mesh_PA_MPJPE'])\n",
+ " if current < best_overall_pa - 1e-4:\n",
+ " best_overall_pa = current\n",
+ " epochs_no_improve = 0\n",
+ " use_ema = ema_hmr['Mesh_PA_MPJPE'] <= raw_hmr['Mesh_PA_MPJPE']\n",
+ " better = ema_model.module if use_ema else model\n",
+ " torch.save({'epoch': epoch, 'model_state_dict': better.state_dict(),\n",
+ " 'val_pa_mpjpe': best_overall_pa, 'source': 'ema' if use_ema else 'raw'},\n",
+ " best_ckpt_path)\n",
+ " print(f\"πŸ’Ύ New best overall: {best_overall_pa:.1f} mm ({'EMA' if use_ema else 'RAW'}) -> {os.path.basename(best_ckpt_path)}\")\n",
+ " else:\n",
+ " epochs_no_improve += 1\n",
+ " print(f\" ⏳ No improvement {epochs_no_improve}/{EARLY_STOP_PATIENCE} (best {best_overall_pa:.1f} mm)\")\n",
+ " if epochs_no_improve >= EARLY_STOP_PATIENCE:\n",
+ " print(f\"\\nπŸ›‘ Early stopping at epoch {epoch+1}: no val improvement in {EARLY_STOP_PATIENCE} epochs.\")\n",
+ " break\n",
+ "\n",
+ " print(f\"\\nπŸŽ‰ Training complete! Best overall Mesh PA-MPJPE: {best_overall_pa:.1f} mm\")\n",
+ " print(f\" best RAW: {best_raw_pa:.1f} mm -> {best_raw_path}\")\n",
+ " print(f\" best EMA: {best_ema_pa:.1f} mm -> {best_ema_path}\")\n",
"\n",
"# Kick off the training!\n",
"if __name__ == \"__main__\":\n",
" import gc\n",
" gc.collect()\n",
" torch.cuda.empty_cache()\n",
- " \n",
- " train_instant_hmr()"
+ " train_instant_hmr()\n"
]
},
{