🧠 两小时从 0 训出你的 64M 大模型。Train your own 64M LLM from scratch in 2 hours.
[fix] keep empty supervised distillation batches finite
Afloat141c830464bd
diff --git a/tests/test_distillation_empty_supervision.py b/tests/test_distillation_empty_supervision.pynew file mode 100644index 0000000..78b51bd--- /dev/null+++ b/tests/test_distillation_empty_supervision.py@@ -0,0 +1,21 @@+import unittest+import torch+from trainer.train_distillation import distillation_loss++class EmptyDistillationTests(unittest.TestCase):+ def test_all_ignored_labels_produce_differentiable_zero(self):+ student = torch.randn(2, 4, 8, requires_grad=True)+ teacher = torch.randn_like(student)+ valid = torch.zeros(2, 4, dtype=torch.bool)+ loss = distillation_loss(student[valid], teacher[valid], temperature=1.5)+ self.assertTrue(bool(loss.isfinite()))+ self.assertEqual(loss.item(), 0.0)+ loss.backward()+ self.assertTrue(torch.equal(student.grad, torch.zeros_like(student)))++ def test_nonempty_identical_logits_keep_zero_loss(self):+ student = torch.tensor([[0.1, 0.2, 0.3]], requires_grad=True)+ loss = distillation_loss(student, student.detach().clone(), temperature=1.5)+ self.assertAlmostEqual(loss.item(), 0.0, places=6)+ loss.backward()+ self.assertTrue(bool(student.grad.isfinite().all()))diff --git a/trainer/train_distillation.py b/trainer/train_distillation.pyindex dce5346..97f7205 100644--- a/trainer/train_distillation.py+++ b/trainer/train_distillation.py@@ -23,6 +23,8 @@ warnings.filterwarnings('ignore')def distillation_loss(student_logits, teacher_logits, temperature=1.0, reduction='batchmean'):+ if student_logits.size(0) == 0 and reduction == 'batchmean':+ return student_logits.sum() * 0.0with torch.no_grad():teacher_probs = F.softmax(teacher_logits / temperature, dim=-1).detach()