Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Pass GradScaler to GKD chunked JSD with an older-Zoo fallback (#13190)

cherry77-cloud8957742a5037
diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.py
index 5cc26b32c4c..4f07c2fd38f 100644
--- a/tests/python/test_gkd_chunked_jsd.py
+++ b/tests/python/test_gkd_chunked_jsd.py
@@ -661,3 +661,54 @@ def test_left_padded_rows_score_as_if_unpadded(shift, monkeypatch):
width = int(keep.sum())
assert torch.equal(liger["labels"][i, :width], labels[i, keep])
assert torch.equal(liger["input_ids"][i, :width], input_ids[i, keep])
+
+
+class _Scaler:
+ def __init__(self, enabled):
+ self.enabled = enabled
+
+ def is_enabled(self):
+ return self.enabled
+
+ def get_scale(self):
+ return 65536.0
+
+
+def _spy_jsd(calls, accepts_scaler):
+ real = rl.distillation_chunked_jsd
+
+ def call(*args, **kwargs):
+ calls.append(dict(kwargs))
+ kwargs.pop("scaler", None)
+ return real(*args, **kwargs)
+
+ if accepts_scaler:
+ return lambda *args, scaler = None, **kwargs: call(*args, scaler = scaler, **kwargs)
+ return lambda *args, **kwargs: call(*args, **kwargs)
+
+
[email protected]("enabled", [True, False, None])
[email protected]("accepts_scaler", [True, False])
+def test_fp16_grad_scaler_reaches_the_chunked_jsd(enabled, accepts_scaler, monkeypatch):
+ # The zoo differentiates inside its forward: an unscaled FP16 grad underflows before scaler.scale(loss).backward().
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 0.0, 2)
+ trainer, inputs = _trainer(0.5, student, teacher), _inputs(vocab)
+ scaler = None if enabled is None else _Scaler(enabled)
+ trainer.accelerator = types.SimpleNamespace(unwrap_model = lambda m: m, scaler = scaler)
+ calls = []
+ monkeypatch.setattr(rl, "distillation_chunked_jsd", _spy_jsd(calls, accepts_scaler))
+ forwards = []
+ student.register_forward_hook(lambda *a: forwards.append(1))
+ got = rl._unsloth_gkd_chunked_loss(
+ trainer, student, inputs, None, {"shift": "prompt", "num_items_in_batch": False}
+ )
+ if enabled and not accepts_scaler:
+ assert got is None and not calls and not forwards
+ assert "unsloth_zoo JSD has no GradScaler support" in trainer._unsloth_gkd_chunked_fallbacks
+ return
+ assert got is not None and len(calls) == 1
+ if enabled:
+ assert calls[0]["scaler"] is scaler
+ else:
+ assert calls[0].get("scaler") is None
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index aa57068e90a..13be72a8e84 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -4450,6 +4450,12 @@ def _unsloth_gkd_chunked_loss(self, model, inputs, num_items_in_batch, layout):
return _unsloth_gkd_note_fallback(self, "use_liger_gkd_loss")
if getattr(self, "is_fsdp_enabled", False) or getattr(self, "is_deepspeed_enabled", False):
return _unsloth_gkd_note_fallback(self, "FSDP / DeepSpeed")
+ jsd_kwargs = {}
+ scaler = getattr(self.accelerator, "scaler", None)
+ if scaler is not None and scaler.is_enabled():
+ if "scaler" not in inspect.signature(distillation_chunked_jsd).parameters:
+ return _unsloth_gkd_note_fallback(self, "unsloth_zoo JSD has no GradScaler support")
+ jsd_kwargs["scaler"] = scaler
try:
missing = any(k not in inputs for k in ("input_ids", "attention_mask", "labels"))
except Exception:
@@ -4573,6 +4579,7 @@ def _unsloth_gkd_chunked_loss(self, model, inputs, num_items_in_batch, layout):
teacher_final_logit_softcapping = teacher_softcap,
# No TRL release passes a temperature to generalized_jsd_loss (GKDConfig.temperature is for sampling).
temperature = 1.0,
+ **jsd_kwargs,
)
try:
self._unsloth_gkd_chunked_calls = getattr(self, "_unsloth_gkd_chunked_calls", 0) + 1