Skip to content

🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.

[fix] keep empty supervised distillation batches finite

Afloat141c830464bd
diff --git a/tests/test_distillation_empty_supervision.py b/tests/test_distillation_empty_supervision.py
new file mode 100644
index 0000000..78b51bd
--- /dev/null
+++ b/tests/test_distillation_empty_supervision.py
@@ -0,0 +1,21 @@
+import unittest
+import torch
+from trainer.train_distillation import distillation_loss
+
+class EmptyDistillationTests(unittest.TestCase):
+ def test_all_ignored_labels_produce_differentiable_zero(self):
+ student = torch.randn(2, 4, 8, requires_grad=True)
+ teacher = torch.randn_like(student)
+ valid = torch.zeros(2, 4, dtype=torch.bool)
+ loss = distillation_loss(student[valid], teacher[valid], temperature=1.5)
+ self.assertTrue(bool(loss.isfinite()))
+ self.assertEqual(loss.item(), 0.0)
+ loss.backward()
+ self.assertTrue(torch.equal(student.grad, torch.zeros_like(student)))
+
+ def test_nonempty_identical_logits_keep_zero_loss(self):
+ student = torch.tensor([[0.1, 0.2, 0.3]], requires_grad=True)
+ loss = distillation_loss(student, student.detach().clone(), temperature=1.5)
+ self.assertAlmostEqual(loss.item(), 0.0, places=6)
+ loss.backward()
+ self.assertTrue(bool(student.grad.isfinite().all()))
diff --git a/trainer/train_distillation.py b/trainer/train_distillation.py
index dce5346..97f7205 100644
--- a/trainer/train_distillation.py
+++ b/trainer/train_distillation.py
@@ -23,6 +23,8 @@ warnings.filterwarnings('ignore')
def distillation_loss(student_logits, teacher_logits, temperature=1.0, reduction='batchmean'):
+ if student_logits.size(0) == 0 and reduction == 'batchmean':
+ return student_logits.sum() * 0.0
with torch.no_grad():
teacher_probs = F.softmax(teacher_logits / temperature, dim=-1).detach()