Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Route GKD distillation through the chunked generalized JSD (#12136)

Daniel Han41b2ff64bd4c
diff --git a/tests/python/test_gkd_chunked_jsd.py b/tests/python/test_gkd_chunked_jsd.py
new file mode 100644
index 00000000000..47c751075d3
--- /dev/null
+++ b/tests/python/test_gkd_chunked_jsd.py
@@ -0,0 +1,578 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""GKD compute_loss routes through the chunked generalized JSD (#11554) and matches TRL."""
+
+from __future__ import annotations
+
+import ast
+import inspect
+import os
+import textwrap
+import types
+
+import pytest
+import torch
+
+os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
+import unsloth.models.rl_replacements as rl
+
+trl = pytest.importorskip("trl")
+try:
+ from trl.experimental.gkd.gkd_trainer import GKDTrainer
+except Exception:
+ from trl.trainer.gkd_trainer import GKDTrainer
+
+if rl.distillation_chunked_jsd is None:
+ pytest.skip("installed unsloth_zoo predates distillation_chunked_jsd", allow_module_level = True)
+
+# The two compute_loss layouts TRL shipped between 0.22.2 and 1.14.0, trimmed to the non-Liger branch.
+PROMPT_LAYOUT = """
+ def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):
+ student_outputs = model(
+ input_ids=inputs["input_ids"],
+ attention_mask=inputs["attention_mask"],
+ )
+ self.teacher_model.eval()
+ with torch.no_grad():
+ teacher_outputs = self.teacher_model(
+ input_ids=inputs["input_ids"],
+ attention_mask=inputs["attention_mask"],
+ )
+ prompt_lengths = inputs["prompts"].shape[1]
+ shifted_student_logits = student_outputs.logits[:, prompt_lengths - 1 : -1, :]
+ shifted_teacher_logits = teacher_outputs.logits[:, prompt_lengths - 1 : -1, :]
+ shifted_labels = inputs["labels"][:, prompt_lengths:]
+ loss = self.generalized_jsd_loss(
+ student_logits=shifted_student_logits,
+ teacher_logits=shifted_teacher_logits,
+ labels=shifted_labels,
+ beta=self.beta,
+ )
+ empty_cache()
+ return (loss, student_outputs) if return_outputs else loss
+"""
+SHIFT_LAYOUT = (
+ PROMPT_LAYOUT.replace(' prompt_lengths = inputs["prompts"].shape[1]\n', "")
+ .replace("[:, prompt_lengths - 1 : -1, :]", "[:, :-1, :]")
+ .replace('inputs["labels"][:, prompt_lengths:]', 'inputs["labels"][:, 1:]')
+ .replace(
+ "beta=self.beta,\n", "beta=self.beta,\n num_items_in_batch=num_items_in_batch,\n"
+ )
+)
+
+
+def test_both_layouts_are_recognised():
+ assert rl._unsloth_gkd_layout(PROMPT_LAYOUT) == {"shift": "prompt", "num_items_in_batch": False}
+ assert rl._unsloth_gkd_layout(SHIFT_LAYOUT) == {"shift": "shift", "num_items_in_batch": True}
+
+
+def test_installed_trl_layout_is_recognised():
+ """Read TRL's file, not the class: importing unsloth has already rebound the class to the patched one."""
+ import importlib.util
+
+ for name in ("trl.trainer.gkd_trainer", "trl.experimental.gkd.gkd_trainer"):
+ try:
+ spec = importlib.util.find_spec(name)
+ except Exception:
+ spec = None
+ if spec is None or spec.origin is None:
+ continue
+ text = open(spec.origin, encoding = "utf-8").read()
+ nodes = [
+ n
+ for n in ast.walk(ast.parse(text))
+ if isinstance(n, ast.FunctionDef) and n.name == "compute_loss"
+ ]
+ if not nodes:
+ continue # a deprecation shim
+ source = ast.get_source_segment(text, nodes[0], padded = True)
+ assert rl._unsloth_gkd_layout(source) is not None, f"{name}.compute_loss is not recognised"
+ return
+ pytest.skip("no GKD trainer in the installed TRL")
+
+
+def test_patched_trainer_uses_the_chunked_loss():
+ patched = GKDTrainer
+ try:
+ import trl.trainer.gkd_trainer as mod
+ patched = mod.GKDTrainer
+ except Exception:
+ pass
+ assert hasattr(patched, "_unsloth_trl_compute_loss"), patched
+ assert rl._unsloth_gkd_jsd_supported(patched)
+
+
[email protected]("source", [PROMPT_LAYOUT, SHIFT_LAYOUT])
+def test_rewrite_parses_and_keeps_trl_body(source):
+ new = rl.gkd_trainer_compute_loss("compute_loss", source)
+ ast.parse("class X:\n" + new)
+ assert new.count("def compute_loss(") == 1
+ assert "def _unsloth_trl_compute_loss(" in new
+ assert "_unsloth_gkd_chunked_loss(" in new
+ assert rl.gkd_trainer_compute_loss("training_step", source) == source
+
+
+ "mutate",
+ [
+ lambda s: s.replace("beta=self.beta,", "beta=self.beta, reduction='sum',"),
+ lambda s: s.replace("beta=self.beta,", "beta=0.3,"),
+ lambda s: s.replace(
+ 'attention_mask=inputs["attention_mask"],\n )',
+ 'attention_mask=inputs["attention_mask"], pixel_values=inputs["pixel_values"],\n )',
+ 1,
+ ),
+ lambda s: s.replace(
+ 'inputs["labels"][:, prompt_lengths:]', 'inputs["labels"][:, prompt_lengths + 1:]'
+ ),
+ lambda s: s.replace("self.generalized_jsd_loss(", "self.other_loss("),
+ lambda s: s.replace("beta=self.beta,", "beta=self.beta, temperature=self.temperature,"),
+ lambda s: s.replace(
+ " empty_cache()\n",
+ " loss = loss + 0.1 * self.generalized_jsd_loss(student_logits=shifted_student_logits, teacher_logits=shifted_teacher_logits, labels=shifted_labels, beta=self.beta)\n empty_cache()\n",
+ ),
+ lambda s: s.replace(
+ "return (loss, student_outputs) if return_outputs else loss",
+ "return (loss * 2, student_outputs) if return_outputs else loss * 2",
+ ),
+ ],
+)
+def test_unrecognised_layouts_are_left_to_trl(mutate):
+ mutated = mutate(PROMPT_LAYOUT)
+ assert mutated != PROMPT_LAYOUT
+ assert rl._unsloth_gkd_layout(mutated) is None
+ assert rl.gkd_trainer_compute_loss("compute_loss", mutated) == mutated
+
+
+class _Head(torch.nn.Module):
+ def __init__(self, head):
+ super().__init__()
+ self.head = head
+
+ def get_output_embeddings(self):
+ return self.head
+
+
+def test_dense_head_guard():
+ assert rl._unsloth_gkd_dense_head(_Head(torch.nn.Linear(8, 32, bias = False))) is not None
+ assert rl._unsloth_gkd_dense_head(_Head(torch.nn.Embedding(32, 8))) is None
+ assert rl._unsloth_gkd_dense_head(_Head(None)) is None
+ int_head = torch.nn.Linear(8, 32, bias = False)
+ int_head.weight = torch.nn.Parameter(torch.zeros(32, 8, dtype = torch.uint8), requires_grad = False)
+ assert rl._unsloth_gkd_dense_head(_Head(int_head)) is None
+ empty = torch.nn.Linear(8, 32, bias = False)
+ empty.weight = torch.nn.Parameter(torch.empty(0)) # ZeRO-3 partitioned
+ assert rl._unsloth_gkd_dense_head(_Head(empty)) is None
+
+
+def test_quantized_and_adapted_heads_fall_back():
+ peft = pytest.importorskip("peft")
+
+ class Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+ self.lm_head = torch.nn.Linear(8, 32, bias = False)
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ lora = peft.get_peft_model(Tiny(), peft.LoraConfig(target_modules = ["lm_head"], r = 2))
+ assert rl._unsloth_gkd_dense_head(lora.base_model.model) is None
+ saved = peft.get_peft_model(
+ Tiny(), peft.LoraConfig(target_modules = ["proj"], modules_to_save = ["lm_head"], r = 2)
+ )
+ assert rl._unsloth_gkd_dense_head(saved.base_model.model) is None
+ try:
+ import bitsandbytes as bnb
+ except Exception:
+ return
+ assert rl._unsloth_gkd_dense_head(_Head(bnb.nn.Linear8bitLt(8, 32, bias = False))) is None
+
+
+class _TinyLM(torch.nn.Module):
+ """Honours UNSLOTH_RETURN_HIDDEN_STATES like Unsloth's compiled forwards: hidden states come back as ``.logits``."""
+
+ __UNSLOTH_SUPPORTS_RETURN_HIDDEN_STATES__ = True
+
+ def __init__(self, vocab, hidden, softcap, seed):
+ super().__init__()
+ g = torch.Generator().manual_seed(seed)
+ self.embed = torch.nn.Parameter(
+ torch.randn(vocab, hidden, generator = g, dtype = torch.float64)
+ )
+ self.mix = torch.nn.Parameter(
+ torch.randn(hidden, hidden, generator = g, dtype = torch.float64) / hidden**0.5
+ )
+ self.lm_head = torch.nn.Linear(hidden, vocab, bias = False, dtype = torch.float64)
+ with torch.no_grad():
+ self.lm_head.weight.copy_(torch.randn(vocab, hidden, generator = g, dtype = torch.float64))
+ self.config = types.SimpleNamespace(final_logit_softcapping = softcap, model_type = "tiny")
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ def forward(self, input_ids, attention_mask):
+ hidden = torch.tanh(self.embed[input_ids] @ self.mix)
+ if os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES", "0") == "1":
+ return types.SimpleNamespace(logits = hidden)
+ logits = self.lm_head(hidden)
+ if self.config.final_logit_softcapping:
+ cap = self.config.final_logit_softcapping
+ logits = cap * torch.tanh(logits / cap)
+ return types.SimpleNamespace(logits = logits)
+
+
+def _trainer(beta, student, teacher):
+ trainer = GKDTrainer.__new__(GKDTrainer)
+ trainer.__dict__.update(
+ beta = beta,
+ temperature = 0.9,
+ use_liger_gkd_loss = False,
+ teacher_model = teacher,
+ accelerator = types.SimpleNamespace(unwrap_model = lambda m: m),
+ )
+ return trainer
+
+
+def _inputs(vocab):
+ g = torch.Generator().manual_seed(0)
+ input_ids = torch.randint(0, vocab, (3, 21), generator = g)
+ labels = input_ids.clone()
+ labels[:, :7] = -100 # prompt
+ labels[1, 17:] = -100 # right padding
+ return {
+ "input_ids": input_ids,
+ "attention_mask": torch.ones_like(input_ids),
+ "labels": labels,
+ "prompts": input_ids[:, :7],
+ }
+
+
[email protected]("beta", [0.0, 0.5, 1.0])
[email protected]("shift", ["prompt", "shift"])
+def test_chunked_loss_matches_trl(beta, shift, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_GKD_CHUNK_SIZE", "8")
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 5.0, 2)
+ trainer, inputs = _trainer(beta, student, teacher), _inputs(vocab)
+ layout = {"shift": shift, "num_items_in_batch": shift == "shift"}
+
+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "0")
+ s, t = student(**{k: inputs[k] for k in ("input_ids", "attention_mask")}).logits, None
+ with torch.no_grad():
+ t = teacher(**{k: inputs[k] for k in ("input_ids", "attention_mask")}).logits
+ if shift == "prompt":
+ s, t, labels = s[:, 6:-1], t[:, 6:-1], inputs["labels"][:, 7:]
+ else:
+ s, t, labels = s[:, :-1], t[:, :-1], inputs["labels"][:, 1:]
+ want = GKDTrainer.generalized_jsd_loss(s, t, labels = labels, beta = beta)
+ want_grads = torch.autograd.grad(want, (student.embed, student.mix))
+
+ got = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ assert got is not None, "chunked path declined a model that honours the flag"
+ assert os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES") == "0", "flag not restored"
+ got_grads = torch.autograd.grad(got, (student.embed, student.mix))
+ torch.testing.assert_close(got.double(), want, rtol = 1e-5, atol = 1e-7)
+ for g, w in zip(got_grads, want_grads):
+ torch.testing.assert_close(g, w, rtol = 1e-4, atol = 1e-7)
+
+
+def test_num_items_in_batch_only_where_trl_uses_it(monkeypatch):
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 0.0, 2)
+ trainer, inputs = _trainer(0.5, student, teacher), _inputs(vocab)
+ local = rl._unsloth_gkd_chunked_loss(
+ trainer,
+ student,
+ inputs,
+ torch.tensor(1000),
+ {"shift": "shift", "num_items_in_batch": False},
+ )
+ scaled = rl._unsloth_gkd_chunked_loss(
+ trainer, student, inputs, torch.tensor(1000), {"shift": "shift", "num_items_in_batch": True}
+ )
+ n = int((inputs["labels"][:, 1:] != -100).sum())
+ torch.testing.assert_close(scaled * 1000, local * n, rtol = 1e-5, atol = 1e-8)
+
+
+def test_fallbacks_return_none(monkeypatch):
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 0.0, 2)
+ inputs = _inputs(vocab)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ monkeypatch.setenv("UNSLOTH_GKD_CHUNKED", "0")
+ assert (
+ rl._unsloth_gkd_chunked_loss(_trainer(0.5, student, teacher), student, inputs, None, layout)
+ is None
+ )
+ monkeypatch.delenv("UNSLOTH_GKD_CHUNKED")
+ liger = _trainer(0.5, student, teacher)
+ liger.use_liger_gkd_loss = True
+ assert rl._unsloth_gkd_chunked_loss(liger, student, inputs, None, layout) is None
+ assert (
+ rl._unsloth_gkd_chunked_loss(_trainer(0.5, student, None), student, inputs, None, layout)
+ is None
+ )
+ assert (
+ rl._unsloth_gkd_chunked_loss(
+ _trainer(0.5, student, _TinyLM(98, 24, 0.0, 2)), student, inputs, None, layout
+ )
+ is None
+ )
+
+ class Custom(GKDTrainer):
+ @staticmethod
+ def generalized_jsd_loss(*args, **kwargs):
+ return 0
+
+ custom = Custom.__new__(Custom)
+ custom.__dict__.update(_trainer(0.5, student, teacher).__dict__)
+ assert rl._unsloth_gkd_chunked_loss(custom, student, inputs, None, layout) is None
+
+
+def test_forward_without_hidden_states_finishes_on_trl_math(monkeypatch):
+ """A teacher that ignores the flag returns real logits; the call must still match TRL, densely."""
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 5.0, 2)
+ type(teacher) # same class, so give this instance a forward that never returns hidden states
+ real_forward = teacher.forward
+
+ def logits_only(input_ids, attention_mask):
+ prior = os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES")
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"
+ try:
+ return real_forward(input_ids, attention_mask)
+ finally:
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = prior
+
+ teacher.forward = logits_only
+ trainer, inputs = _trainer(0.5, student, teacher), _inputs(vocab)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ got = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "0")
+ s = student(inputs["input_ids"], inputs["attention_mask"]).logits[:, :-1]
+ with torch.no_grad():
+ t = real_forward(inputs["input_ids"], inputs["attention_mask"]).logits[:, :-1]
+ want = GKDTrainer.generalized_jsd_loss(s, t, labels = inputs["labels"][:, 1:], beta = 0.5)
+ torch.testing.assert_close(got, want, rtol = 1e-6, atol = 1e-9)
+
+
+def test_generated_items_are_self_contained():
+ """RL_PRE_ITEMS carries every helper the rewritten compute_loss calls into the generated module."""
+ pre = "\n".join(rl.RL_PRE_ITEMS["gkd_trainer"])
+ for name in (
+ "_unsloth_gkd_chunked_loss",
+ "_unsloth_gkd_dense_head",
+ "_unsloth_gkd_logit_transforms",
+ "_unsloth_gkd_project",
+ "_unsloth_gkd_canonical",
+ "_unsloth_gkd_jsd_supported",
+ "_unsloth_gkd_note_fallback",
+ "_unsloth_gkd_chunk_size",
+ "_unsloth_grpo_returns_hidden_states",
+ "_unsloth_grpo_hidden_states_signal",
+ "_unsloth_get_model_config",
+ ):
+ assert f"def {name}(" in pre, name
+ assert "from unsloth_zoo.rl_replacements import distillation_chunked_jsd" in pre
+ assert rl.gkd_trainer_compute_loss in rl.RL_FUNCTIONS["gkd_trainer"]
+
+
+def _jsd_source():
+ """TRL's own ``generalized_jsd_loss`` from the installed file (the class may already be the generated one)."""
+ import importlib.util
+
+ for name in ("trl.experimental.gkd.gkd_trainer", "trl.trainer.gkd_trainer"):
+ try:
+ spec = importlib.util.find_spec(name)
+ except Exception:
+ spec = None
+ if spec is None or spec.origin is None:
+ continue
+ text = open(spec.origin, encoding = "utf-8").read()
+ for node in ast.walk(ast.parse(text)):
+ if isinstance(node, ast.FunctionDef) and node.name == "generalized_jsd_loss":
+ return textwrap.dedent(ast.get_source_segment(text, node, padded = True))
+ pytest.skip("no GKD trainer in the installed TRL")
+
+
+def _jsd_class(source, tmp_path, name):
+ import importlib.util
+
+ body = (
+ "import torch\nimport torch.nn.functional as F\nclass GKDTrainer:\n @staticmethod\n"
+ + textwrap.indent(source, " ")
+ )
+ path = tmp_path / f"{name}.py"
+ path.write_text(body)
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module.GKDTrainer
+
+
+def test_installed_generalized_jsd_is_accepted(tmp_path):
+ assert rl._unsloth_gkd_jsd_supported(_jsd_class(_jsd_source(), tmp_path, "gkd_jsd_ok"))
+
+
+ "old, new",
+ [
+ (
+ "jsd = beta * kl_teacher + (1 - beta) * kl_student",
+ "jsd = (beta * kl_teacher + (1 - beta) * kl_student) * temperature ** 2",
+ ),
+ ("jsd = jsd[mask]", "jsd = jsd[mask] + 0.0"),
+ (
+ "student_log_probs = F.log_softmax(student_logits, dim=-1)",
+ "student_log_probs = F.log_softmax(student_logits.float(), dim=-1)",
+ ),
+ ("jsd_sum / num_items_in_batch", "jsd_sum / num_items_in_batch / jsd.size(-1)"),
+ ("return jsd_sum / num_items_in_batch", "return jsd_sum"),
+ ],
+)
+def test_changed_generalized_jsd_is_left_to_trl(old, new, tmp_path):
+ source = _jsd_source()
+ if old not in source:
+ pytest.skip("line not in this TRL release")
+ mutated = source.replace(old, new)
+ name = f"gkd_jsd_mut_{abs(hash((old, new)))}"
+ assert not rl._unsloth_gkd_jsd_supported(_jsd_class(mutated, tmp_path, name))
+
+
+def test_fsdp_deepspeed_and_missing_inputs_fall_back():
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 0.0, 2)
+ layout = {"shift": "prompt", "num_items_in_batch": False}
+ for flag in ("is_fsdp_enabled", "is_deepspeed_enabled"):
+ trainer = _trainer(0.5, student, teacher)
+ setattr(trainer, flag, True)
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(vocab), None, layout) is None
+ assert trainer._unsloth_gkd_chunked_fallbacks == {"FSDP / DeepSpeed": 1}
+ inputs = _inputs(vocab)
+ inputs.pop("prompts")
+ assert (
+ rl._unsloth_gkd_chunked_loss(_trainer(0.5, student, teacher), student, inputs, None, layout)
+ is None
+ )
+
+
+def test_batch_encoding_inputs_take_the_chunked_path():
+ from transformers import BatchEncoding
+
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 0.0, 2)
+ layout = {"shift": "prompt", "num_items_in_batch": False}
+ got = rl._unsloth_gkd_chunked_loss(
+ _trainer(0.5, student, teacher), student, BatchEncoding(_inputs(vocab)), None, layout
+ )
+ want = rl._unsloth_gkd_chunked_loss(
+ _trainer(0.5, student, teacher), student, _inputs(vocab), None, layout
+ )
+ assert got is not None
+ torch.testing.assert_close(got, want)
+
+
+def test_dense_fallback_projects_on_the_head_device():
+ """A dispatched model can leave its final hidden states on another device than a tied lm_head."""
+ head = torch.nn.Linear(4, 5).to("meta")
+ logits = rl._unsloth_gkd_project(torch.randn(2, 3, 4), head, 0.5, 30.0)
+ assert logits.device == head.weight.device and logits.shape == (2, 3, 5)
+
+
+ not torch.cuda.is_available(),
+ reason = "needs a second device to split hidden states from the head",
+)
+def test_chunked_loss_with_heads_on_another_device():
+ """A dispatched model can return hidden states from one device while a tied lm_head sits on another."""
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 5.0, 2)
+ trainer, inputs = _trainer(0.5, student, teacher), _inputs(vocab)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ expected = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ student.lm_head.cuda()
+ teacher.lm_head.cuda()
+ loss = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ loss.backward()
+ torch.testing.assert_close(loss.cpu().double(), expected.double(), rtol = 1e-5, atol = 1e-7)
+ assert student.mix.grad is not None and student.mix.grad.device.type == "cpu"
+
+
+def test_offloaded_meta_heads_fall_back():
+ """Accelerate CPU / disk offload keeps weights on meta between forwards; only the hooked forward can read them."""
+ model = _TinyLM(97, 16, 0.0, 1)
+ assert rl._unsloth_gkd_dense_head(model) is model.lm_head
+ model.lm_head.to("meta")
+ assert rl._unsloth_gkd_dense_head(model) is None
+
+
+def test_ddp_find_unused_with_trainable_head_falls_back():
+ """DDP(find_unused_parameters=True) marks a head skipped in forward as unused, then its grad hook fires twice."""
+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 5.0, 2)
+ ddp = torch.nn.parallel.DistributedDataParallel.__new__(
+ torch.nn.parallel.DistributedDataParallel
+ )
+ torch.nn.Module.__init__(ddp)
+ ddp.module, ddp.find_unused_parameters = student, True
+ ddp.forward = student.forward # the stub has no process group
+ trainer = _trainer(0.5, student, teacher)
+ trainer.accelerator = types.SimpleNamespace(unwrap_model = lambda m: getattr(m, "module", m))
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ assert rl._unsloth_gkd_chunked_loss(trainer, ddp, _inputs(97), None, layout) is None
+ assert any(
+ "find_unused_parameters" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks
+ )
+ student.lm_head.weight.requires_grad_(False)
+ assert rl._unsloth_gkd_chunked_loss(trainer, ddp, _inputs(97), None, layout) is not None
+
+
+def test_minicpm3_scales_hidden_states_before_the_head_so_falls_back():
+ """MiniCPM3 divides hidden states by logits_scaling before lm_head; hidden_states[-1] predates that."""
+ student, teacher = _TinyLM(97, 16, 0.0, 1), _TinyLM(97, 24, 5.0, 2)
+ teacher.config.model_type = "minicpm3"
+ trainer = _trainer(0.5, student, teacher)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ assert rl._unsloth_gkd_chunked_loss(trainer, student, _inputs(97), None, layout) is None
+ assert any("minicpm3" in reason for reason in trainer._unsloth_gkd_chunked_fallbacks)
+
+
+ not torch.cuda.is_available(),
+ reason = "needs a second device to split hidden states from the head",
+)
+def test_dense_fallback_colocates_with_the_labels():
+ """One forward returns logits on the input device, the other hidden states projected on its head's device."""
+ vocab = 97
+ student, teacher = _TinyLM(vocab, 16, 0.0, 1), _TinyLM(vocab, 24, 5.0, 2)
+ real_forward = teacher.forward
+
+ def logits_only(input_ids, attention_mask):
+ prior = os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES")
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "0"
+ try:
+ return real_forward(input_ids, attention_mask)
+ finally:
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = prior
+
+ teacher.forward = logits_only
+ trainer, inputs = _trainer(0.5, student, teacher), _inputs(vocab)
+ layout = {"shift": "shift", "num_items_in_batch": False}
+ want = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ student.lm_head.cuda()
+ got = rl._unsloth_gkd_chunked_loss(trainer, student, inputs, None, layout)
+ assert got.device == inputs["labels"].device
+ torch.testing.assert_close(got, want, rtol = 1e-6, atol = 1e-9)
+
+
+def test_prediction_head_before_the_decoder_falls_back():
+ """ModernBERT-decoder / RoBERTa-style: logits = decoder(lm_head(hidden)), so hidden states alone miss lm_head."""
+ model = _TinyLM(97, 16, 0.0, 1)
+ decoder = model.lm_head
+ model.lm_head = torch.nn.Sequential(
+ torch.nn.Linear(16, 16, dtype = torch.float64), torch.nn.GELU()
+ )
+ model.decoder = decoder
+ model.get_output_embeddings = lambda: model.decoder
+ assert rl._unsloth_gkd_dense_head(model) is None
diff --git a/tests/python/test_grpo_ddp_model_config.py b/tests/python/test_grpo_ddp_model_config.py
index 4078029b2f2..f079a6062a7 100644
--- a/tests/python/test_grpo_ddp_model_config.py
+++ b/tests/python/test_grpo_ddp_model_config.py
@@ -23,8 +23,8 @@ def test_grpo_logit_scaling_uses_model_config_helper():
# Whitespace-insensitive: the formatter wraps one of the two calls and not the other.
flat = "".join(src.split())
assert (
- flat.count("=_unsloth_resolve_logit_scales(model_config)") == 2
- ), "both fallback arms must resolve the scales from the unwrapped config"
+ flat.count("=_unsloth_resolve_logit_scales(model_config)") == 3
+ ), "every fallback arm (two GRPO, one GKD) must resolve the scales from the unwrapped config"
assert "_unsloth_resolve_logit_scales(model)" not in flat
assert src.count("model_config = _unsloth_get_model_config(model)") >= 2
# Helper source is injected into the compiled GRPO trainer.
diff --git a/tests/python/test_grpo_logit_transform_fallback.py b/tests/python/test_grpo_logit_transform_fallback.py
index 36002865ebc..7c39b1dca96 100644
--- a/tests/python/test_grpo_logit_transform_fallback.py
+++ b/tests/python/test_grpo_logit_transform_fallback.py
@@ -44,9 +44,10 @@ def _fallback_blocks():
BLOCKS = _fallback_blocks()
-def test_both_grpo_call_sites_were_found():
- """A rename would otherwise make this whole file silently vacuous."""
- assert len(BLOCKS) == 2, f"expected two fallback arms, found {sorted(BLOCKS)}"
+def test_every_call_site_was_found():
+ """A rename would otherwise make this whole file silently vacuous. Two GRPO bodies plus the GKD chunked loss."""
+ assert len(BLOCKS) == 3, f"expected three fallback arms, found {sorted(BLOCKS)}"
+ assert any(site.startswith("_unsloth_gkd_logit_transforms:") for site in BLOCKS), sorted(BLOCKS)
class _Cfg:
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index ef4cf299ff0..403ad71fb25 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1822,15 +1822,15 @@ def _install_grpo_hidden_states_forward_wrapper(model):
return True
-def _wrap_grpo_hidden_states_fallback(trainer_cls):
+def _wrap_grpo_hidden_states_fallback(trainer_cls, attributes = ("model", "ref_model")):
original_init = trainer_cls.__init__
if getattr(original_init, "_unsloth_grpo_hidden_states_init_wrapped", False):
return
def wrapped_init(self, *args, **kwargs):
original_init(self, *args, **kwargs)
- _install_grpo_hidden_states_forward_wrapper(getattr(self, "model", None))
- _install_grpo_hidden_states_forward_wrapper(getattr(self, "ref_model", None))
+ for attribute in attributes:
+ _install_grpo_hidden_states_forward_wrapper(getattr(self, attribute, None))
wrapped_init._unsloth_grpo_hidden_states_init_wrapped = True
trainer_cls.__init__ = wrapped_init
@@ -3391,6 +3391,16 @@ def _patch_trl_rl_trainers_impl(trainer_file = "grpo_trainer"):
logger.info(
f"Unsloth: Could not wrap GRPO hidden-state fallback for {RLTrainer_name}: {e}"
)
+ if trainer_file == "gkd_trainer" and "_unsloth_trl_compute_loss" in RLTrainer_source:
+ try:
+ _wrap_grpo_hidden_states_fallback(
+ getattr(created_module, f"Unsloth{RLTrainer_name}"),
+ attributes = ("model", "teacher_model"),
+ )
+ except Exception as e:
+ logger.info(
+ f"Unsloth: Could not wrap GKD hidden-state fallback for {RLTrainer_name}: {e}"
+ )
def patch_functions(RLTrainer, trainer_file, RLTrainer_name, all_imports, imports):
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index eb594c0e04d..183b42f9aa7 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -833,7 +833,7 @@ RL_FUNCTIONS["sft_trainer"].append(sft_trainer_prepare_dataset)
# Ignore mean_token_accuracy since it needs logits; it is overridden with our version.
def sft_trainer_compute_loss(function_name, function):
- if function_name != "compute_loss":
+ if function_name != "compute_loss" or "_unsloth_trl_compute_loss" in function:
return function
def compute_loss(
@@ -2951,7 +2951,7 @@ RL_PRE_ITEMS["grpo_trainer"].append(
def grpo_trainer_compute_loss(function_name, function):
- if function_name != "compute_loss":
+ if function_name != "compute_loss" or "_unsloth_trl_compute_loss" in function:
return function
def compute_loss(
@@ -3800,3 +3800,437 @@ def vllm_generation_init_patch():
RL_ADDITIONAL_FUNCTIONS["vllm_generation"].append(vllm_generation_init_patch)
+
+
+# GKD chunked JSD (#11554): anything not reproduced exactly returns None and TRL's own compute_loss runs.
+try:
+ from unsloth_zoo.rl_replacements import distillation_chunked_jsd
+except Exception:
+ distillation_chunked_jsd = None
+
+
+def _unsloth_gkd_canonical(source):
+ """``ast.unparse`` without docstrings/comments so checks survive reformatting; ``None`` if unparsable."""
+ import ast as _ast
+ import textwrap as _textwrap
+
+ try:
+ tree = _ast.parse(_textwrap.dedent(source))
+ except Exception:
+ return None
+ for node in _ast.walk(tree):
+ if isinstance(node, (_ast.FunctionDef, _ast.AsyncFunctionDef)) and node.body:
+ first = node.body[0]
+ if (
+ isinstance(first, _ast.Expr)
+ and isinstance(getattr(first, "value", None), _ast.Constant)
+ and isinstance(first.value.value, str)
+ ):
+ node.body = node.body[1:] or [_ast.Pass()]
+ try:
+ return _ast.unparse(tree)
+ except Exception:
+ return None
+
+
+def _unsloth_gkd_layout(source):
+ """Which TRL ``GKDTrainer.compute_loss`` this is, or ``None`` when it is not one we reproduce exactly.
+ ``prompt`` (TRL < 1.7) or ``shift`` (TRL >= 1.7); any extra input, loss knob or term must fall back.
+ """
+ source = _unsloth_gkd_canonical(source)
+ if source is None:
+ return None
+ call_args = "(input_ids=inputs['input_ids'], attention_mask=inputs['attention_mask'])"
+ if len(re.findall(r"(?<![\w.])model" + re.escape(call_args), source)) != 1:
+ return None
+ if source.count("self.teacher_model" + call_args) != 1:
+ return None
+ if "return (loss, student_outputs) if return_outputs else loss" not in source:
+ return None
+ calls = re.findall(
+ r"self\.generalized_jsd_loss\(student_logits=shifted_student_logits, "
+ r"teacher_logits=shifted_teacher_logits, labels=shifted_labels, beta=self\.beta"
+ r"(, num_items_in_batch=num_items_in_batch)?\)",
+ source,
+ )
+ if len(calls) != 1 or source.count("self.generalized_jsd_loss(") != 1:
+ return None
+ prompt_layout = (
+ "prompt_lengths = inputs['prompts'].shape[1]",
+ "shifted_student_logits = student_outputs.logits[:, prompt_lengths - 1:-1, :]",
+ "shifted_teacher_logits = teacher_outputs.logits[:, prompt_lengths - 1:-1, :]",
+ "shifted_labels = inputs['labels'][:, prompt_lengths:]",
+ )
+ shift_layout = (
+ "shifted_student_logits = student_outputs.logits[:, :-1, :]",
+ "shifted_teacher_logits = teacher_outputs.logits[:, :-1, :]",
+ "shifted_labels = inputs['labels'][:, 1:]",
+ )
+ is_prompt = all(x in source for x in prompt_layout)
+ is_shift = all(x in source for x in shift_layout)
+ if is_prompt == is_shift:
+ return None
+ return {"shift": "prompt" if is_prompt else "shift", "num_items_in_batch": bool(calls[0])}
+
+
+def _unsloth_gkd_dense_head(model):
+ """The output head when it is a plain dense ``[vocab, hidden]`` projection the chunked loss can read directly.
+ Rejects bnb heads (``nn.Linear`` subclasses with packed weights), PEFT heads and ZeRO-3 partitioned weights.
+ """
+ get_output_embeddings = getattr(model, "get_output_embeddings", None)
+ if not callable(get_output_embeddings):
+ return None
+ try:
+ head = get_output_embeddings()
+ except Exception:
+ return None
+ if type(head) is not torch.nn.Linear:
+ return None
+ # An lm_head that is not the output embeddings runs before them (ModernBERT decoder, RoBERTa-style heads).
+ lm_head = getattr(model, "lm_head", None)
+ if isinstance(lm_head, torch.nn.Module) and lm_head is not head:
+ return None
+ weight = getattr(head, "weight", None)
+ if not isinstance(weight, torch.Tensor) or weight.dim() != 2 or weight.numel() == 0:
+ return None
+ if weight.dtype not in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
+ return None
+ if type(weight) not in (torch.Tensor, torch.nn.Parameter):
+ return None
+ bias = getattr(head, "bias", None)
+ if bias is not None and (bias.dim() != 1 or bias.shape[0] != weight.shape[0]):
+ return None
+ if weight.is_meta or (bias is not None and bias.is_meta):
+ return None
+ return head
+
+
+def _unsloth_gkd_logit_transforms(model):
+ """``(scale, softcap)`` exactly as the model's own forward applies them to its logits."""
+ model_config = _unsloth_get_model_config(model)
+ if detect_logit_transforms is not None:
+ _transforms = detect_logit_transforms(model_config)
+ logit_softcapping = _transforms["logit_softcapping"]
+ logit_scale_multiply = _transforms["logit_scale_multiply"]
+ logit_scale_divide = _transforms["logit_scale_divide"]
+ else:
+ logit_softcapping = _unsloth_get_final_logit_softcapping(model)
+ logit_scale_multiply, logit_scale_divide = _unsloth_resolve_logit_scales(model_config)
+ scale = 1.0
+ if logit_scale_multiply:
+ scale = scale * float(logit_scale_multiply)
+ if logit_scale_divide:
+ scale = scale / float(logit_scale_divide)
+ return scale, float(logit_softcapping or 0.0)
+
+
+def _unsloth_gkd_project(hidden_states, head, scale, softcap):
+ """Full logits from hidden states, for the rare call where only one of the two forwards honoured the flag."""
+ hidden_states = hidden_states.to(device = head.weight.device, dtype = head.weight.dtype)
+ logits = torch.nn.functional.linear(hidden_states, head.weight, head.bias)
+ if scale != 1.0:
+ logits = logits * scale
+ if softcap:
+ logits = softcap * torch.tanh(logits / softcap)
+ return logits
+
+
+def _unsloth_gkd_jsd_supported(trainer_class):
+ """Is ``trainer_class.generalized_jsd_loss`` one of TRL's own generalized JSDs, unchanged? Overrides keep the dense loss."""
+ cache = trainer_class.__dict__.get("_unsloth_gkd_jsd_supported_cache", None)
+ if cache is not None:
+ return cache
+ required = (
+ "student_logits = student_logits / temperature",
+ "teacher_logits = teacher_logits / temperature",
+ "student_log_probs = F.log_softmax(student_logits, dim=-1)",
+ "teacher_log_probs = F.log_softmax(teacher_logits, dim=-1)",
+ "if beta == 0:",
+ "jsd = F.kl_div(student_log_probs, teacher_log_probs, reduction='none', log_target=True)",
+ "elif beta == 1:",
+ "jsd = F.kl_div(teacher_log_probs, student_log_probs, reduction='none', log_target=True)",
+ "kl_teacher = F.kl_div(mixture_log_probs, teacher_log_probs, reduction='none', log_target=True)",
+ "kl_student = F.kl_div(mixture_log_probs, student_log_probs, reduction='none', log_target=True)",
+ "jsd = beta * kl_teacher + (1 - beta) * kl_student",
+ "mask = labels != -100",
+ "jsd = jsd[mask]",
+ "if reduction == 'batchmean':",
+ )
+ known = set(required) | {
+ "@staticmethod",
+ "def generalized_jsd_loss(student_logits, teacher_logits, labels=None, beta=0.5, temperature=1.0, reduction='batchmean'):",
+ "def generalized_jsd_loss(student_logits, teacher_logits, labels=None, beta=0.5, temperature=1.0, reduction='batchmean', num_items_in_batch=None):",
+ "else:",
+ "if labels is not None:",
+ "beta = torch.tensor(beta, dtype=student_log_probs.dtype)",
+ "beta = torch.tensor(beta, dtype=student_log_probs.dtype, device=student_log_probs.device)",
+ "mixture_log_probs = torch.logsumexp(torch.stack([student_log_probs + torch.log(1 - beta), teacher_log_probs + torch.log(beta)]), dim=0)",
+ "mixture_log_probs = torch.logsumexp(torch.stack([student_log_probs + torch.log1p(-beta), teacher_log_probs + torch.log(beta)]), dim=0)",
+ "if num_items_in_batch is not None:",
+ "jsd_sum = jsd.sum()",
+ "if isinstance(num_items_in_batch, torch.Tensor):",
+ "num_items_in_batch = num_items_in_batch.to(jsd_sum.device)",
+ "return jsd_sum / num_items_in_batch",
+ "return jsd.sum() / mask.sum() if labels is not None else jsd.sum() / (jsd.size(0) * jsd.size(1))",
+ "return jsd.sum() / mask.sum() if labels is not None else jsd.sum() / jsd.size(0)",
+ "denom = mask.sum().clamp_min(1) if labels is not None else max(jsd.size(0), 1)",
+ "return jsd.sum() / denom",
+ "elif reduction == 'sum':",
+ "return jsd.sum()",
+ "elif reduction == 'mean':",
+ "return jsd.mean()",
+ "return jsd",
+ }
+ ok = False
+ try:
+ source = _unsloth_gkd_canonical(inspect.getsource(trainer_class.generalized_jsd_loss))
+ lines = [line.strip() for line in source.splitlines() if line.strip()]
+ signature = next((line for line in lines if line.startswith("def ")), "")
+ ok = (
+ all(line in known for line in lines)
+ and all(line in lines for line in required)
+ # a num_items_in_batch parameter must come with its reduction, and vice versa
+ and (
+ ("num_items_in_batch=None" in signature)
+ == ("return jsd_sum / num_items_in_batch" in lines)
+ )
+ )
+ except Exception:
+ ok = False
+ try:
+ setattr(trainer_class, "_unsloth_gkd_jsd_supported_cache", ok)
+ except Exception:
+ pass
+ return ok
+
+
+def _unsloth_gkd_note_fallback(trainer, reason):
+ """Count why the chunked path declined (``trainer._unsloth_gkd_chunked_fallbacks``), log each reason once."""
+ try:
+ fallbacks = trainer.__dict__.setdefault("_unsloth_gkd_chunked_fallbacks", {})
+ except Exception:
+ return None
+ if reason not in fallbacks:
+ try:
+ from unsloth_zoo.log import logger as _gkd_logger
+ _gkd_logger.info(
+ f"Unsloth: GKD chunked JSD not used ({reason}); using TRL's dense loss."
+ )
+ except Exception:
+ pass
+ fallbacks[reason] = fallbacks.get(reason, 0) + 1
+ return None
+
+
+def _unsloth_gkd_chunk_size(vocab_size):
+ """Rows per chunk: ``UNSLOTH_GKD_CHUNK_SIZE`` when set, else about 2**26 logits per chunk, bounded to [64, 1024]."""
+ requested = os.environ.get("UNSLOTH_GKD_CHUNK_SIZE", "")
+ if requested.strip().isdigit() and int(requested) > 0:
+ return int(requested)
+ rows = (1 << 26) // max(int(vocab_size), 1)
+ return int(min(1024, max(64, 1 << (max(rows, 1).bit_length() - 1))))
+
+
+def _unsloth_gkd_chunked_loss(self, model, inputs, num_items_in_batch, layout):
+ """TRL's GKD loss without either full logits tensor, or ``None`` to run TRL's own ``compute_loss``."""
+ if layout is None:
+ return None
+ if distillation_chunked_jsd is None:
+ return _unsloth_gkd_note_fallback(self, "unsloth_zoo has no distillation_chunked_jsd")
+ if os.environ.get("UNSLOTH_GKD_CHUNKED", "1").lower() in ("0", "false", "no", "off"):
+ return _unsloth_gkd_note_fallback(self, "UNSLOTH_GKD_CHUNKED=0")
+ if getattr(self, "use_liger_gkd_loss", False):
+ return _unsloth_gkd_note_fallback(self, "use_liger_gkd_loss")
+ if getattr(self, "is_fsdp_enabled", False) or getattr(self, "is_deepspeed_enabled", False):
+ return _unsloth_gkd_note_fallback(self, "FSDP / DeepSpeed")
+ try:
+ missing = any(k not in inputs for k in ("input_ids", "attention_mask", "labels"))
+ except Exception:
+ missing = True
+ if missing:
+ return _unsloth_gkd_note_fallback(self, "inputs")
+ if layout["shift"] == "prompt" and "prompts" not in inputs:
+ return _unsloth_gkd_note_fallback(self, "inputs")
+ teacher_model = getattr(self, "teacher_model", None)
+ if teacher_model is None:
+ return _unsloth_gkd_note_fallback(self, "no teacher_model")
+ if not _unsloth_gkd_jsd_supported(type(self)):
+ return _unsloth_gkd_note_fallback(self, "generalized_jsd_loss is not TRL's")
+ try:
+ unwrapped_student = self.accelerator.unwrap_model(model)
+ unwrapped_teacher = self.accelerator.unwrap_model(teacher_model)
+ except Exception:
+ return _unsloth_gkd_note_fallback(self, "unwrap_model")
+ student_head = _unsloth_gkd_dense_head(unwrapped_student)
+ teacher_head = _unsloth_gkd_dense_head(unwrapped_teacher)
+ if student_head is None or teacher_head is None:
+ return _unsloth_gkd_note_fallback(self, "output head is not a dense nn.Linear")
+ if student_head.weight.shape[0] != teacher_head.weight.shape[0]:
+ return _unsloth_gkd_note_fallback(self, "vocab mismatch")
+ # MiniCPM3 divides hidden states by logits_scaling before lm_head; a wrapped forward's hidden_states[-1] predates it.
+ for unwrapped in (unwrapped_student, unwrapped_teacher):
+ for config in _unsloth_text_configs(_unsloth_get_model_config(unwrapped)):
+ if getattr(config, "model_type", None) == "minicpm3":
+ return _unsloth_gkd_note_fallback(
+ self, "minicpm3 scales hidden states before the head"
+ )
+ # DDP(find_unused_parameters=True) marks a head skipped in forward as unused, then its grad hook fires twice.
+ if getattr(model, "find_unused_parameters", False) and any(
+ p is not None and p.requires_grad for p in (student_head.weight, student_head.bias)
+ ):
+ return _unsloth_gkd_note_fallback(self, "DDP find_unused_parameters with a trainable head")
+
+ prior_hidden_states = os.environ.get("UNSLOTH_RETURN_HIDDEN_STATES")
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"
+ try:
+ student_outputs = model(
+ input_ids = inputs["input_ids"],
+ attention_mask = inputs["attention_mask"],
+ )
+ teacher_model.eval()
+ with torch.no_grad():
+ teacher_outputs = teacher_model(
+ input_ids = inputs["input_ids"],
+ attention_mask = inputs["attention_mask"],
+ )
+ finally:
+ if prior_hidden_states is None:
+ os.environ.pop("UNSLOTH_RETURN_HIDDEN_STATES", None)
+ else:
+ os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = prior_hidden_states
+
+ student_states = student_outputs.logits
+ teacher_states = teacher_outputs.logits
+ if layout["shift"] == "prompt":
+ prompt_lengths = inputs["prompts"].shape[1]
+ student_states = student_states[:, prompt_lengths - 1 : -1, :]
+ teacher_states = teacher_states[:, prompt_lengths - 1 : -1, :]
+ shifted_labels = inputs["labels"][:, prompt_lengths:]
+ else:
+ student_states = student_states[:, :-1, :]
+ teacher_states = teacher_states[:, :-1, :]
+ shifted_labels = inputs["labels"][:, 1:]
+
+ student_scale, student_softcap = _unsloth_gkd_logit_transforms(unwrapped_student)
+ teacher_scale, teacher_softcap = _unsloth_gkd_logit_transforms(unwrapped_teacher)
+ items = num_items_in_batch if layout["num_items_in_batch"] else None
+
+ student_hidden = _unsloth_grpo_returns_hidden_states(
+ unwrapped_student, student_states, student_head.weight
+ )
+ teacher_hidden = _unsloth_grpo_returns_hidden_states(
+ unwrapped_teacher, teacher_states, teacher_head.weight
+ )
+ if not (student_hidden and teacher_hidden):
+ if student_hidden:
+ student_states = _unsloth_gkd_project(
+ student_states, student_head, student_scale, student_softcap
+ )
+ if teacher_hidden:
+ with torch.no_grad():
+ teacher_states = _unsloth_gkd_project(
+ teacher_states, teacher_head, teacher_scale, teacher_softcap
+ )
+ _unsloth_gkd_note_fallback(self, "a forward returned logits, not hidden states")
+ # TRL's dense path gets both logits on the input device (accelerate's top-level hook), beside the labels.
+ student_states = student_states.to(shifted_labels.device)
+ teacher_states = teacher_states.to(shifted_labels.device)
+ extra = {}
+ if layout["num_items_in_batch"]:
+ extra["num_items_in_batch"] = num_items_in_batch
+ return self.generalized_jsd_loss(
+ student_logits = student_states,
+ teacher_logits = teacher_states,
+ labels = shifted_labels,
+ beta = self.beta,
+ **extra,
+ )
+
+ # A dispatched model can return hidden states off its head's device; the zoo's accumulators follow the states.
+ student_states = student_states.to(student_head.weight.device)
+ teacher_states = teacher_states.to(teacher_head.weight.device)
+ loss, _entropy_sum, _n_valid = distillation_chunked_jsd(
+ student_states,
+ teacher_states.detach(),
+ student_head.weight,
+ teacher_head.weight.detach(),
+ shifted_labels != -100,
+ beta = float(self.beta),
+ chunk_size = _unsloth_gkd_chunk_size(student_head.weight.shape[0]),
+ num_items_in_batch = items,
+ student_lm_head_bias = student_head.bias,
+ teacher_lm_head_bias = None if teacher_head.bias is None else teacher_head.bias.detach(),
+ student_logit_scale = student_scale,
+ teacher_logit_scale = teacher_scale,
+ student_final_logit_softcapping = student_softcap,
+ teacher_final_logit_softcapping = teacher_softcap,
+ # No TRL release passes a temperature to generalized_jsd_loss (GKDConfig.temperature is for sampling).
+ temperature = 1.0,
+ )
+ try:
+ self._unsloth_gkd_chunked_calls = getattr(self, "_unsloth_gkd_chunked_calls", 0) + 1
+ except Exception:
+ pass
+ return loss
+
+
+def gkd_trainer_compute_loss(function_name, function):
+ """Route the generated ``compute_loss`` through the chunked loss, keeping TRL's body as the fallback."""
+ if function_name != "compute_loss" or "_unsloth_trl_compute_loss" in function:
+ return function
+ layout = _unsloth_gkd_layout(function)
+ if layout is None or distillation_chunked_jsd is None:
+ return function
+ renamed, n = re.subn(
+ r"\bdef compute_loss\(", "def _unsloth_trl_compute_loss(", function, count = 1
+ )
+ if n != 1:
+ return function
+ indent = re.search(r"^([ \t]*)def compute_loss\(", function, flags = re.MULTILINE).group(1)
+ wrapper = textwrap.indent(
+ "def compute_loss(self, model, inputs, return_outputs=False, num_items_in_batch=None):\n"
+ " # Unsloth: chunked generalized JSD over hidden states; TRL's own loss below is the fallback.\n"
+ " if not return_outputs:\n"
+ " loss = _unsloth_gkd_chunked_loss(\n"
+ f" self, model, inputs, num_items_in_batch, {layout!r},\n"
+ " )\n"
+ " if loss is not None:\n"
+ " return loss\n"
+ " return self._unsloth_trl_compute_loss(\n"
+ " model, inputs, return_outputs=return_outputs, num_items_in_batch=num_items_in_batch,\n"
+ " )\n",
+ indent,
+ )
+ return wrapper + "\n" + renamed
+
+
+RL_FUNCTIONS["gkd_trainer"].append(gkd_trainer_compute_loss)
+for _gkd_item in (
+ _unsloth_get_model_config,
+ _unsloth_text_configs,
+ _unsloth_get_final_logit_softcapping,
+ _unsloth_resolve_logit_scales,
+ _unsloth_grpo_returns_hidden_states,
+ _unsloth_grpo_hidden_states_signal,
+ _unsloth_gkd_canonical,
+ _unsloth_gkd_jsd_supported,
+ _unsloth_gkd_note_fallback,
+ _unsloth_gkd_dense_head,
+ _unsloth_gkd_logit_transforms,
+ _unsloth_gkd_project,
+ _unsloth_gkd_chunk_size,
+ _unsloth_gkd_chunked_loss,
+):
+ RL_PRE_ITEMS["gkd_trainer"].append(inspect.getsource(_gkd_item))
+RL_PRE_ITEMS["gkd_trainer"].append(
+ "import re\n"
+ "import inspect\n"
+ "try:\n"
+ " from unsloth_zoo.device_map_planner import detect_logit_transforms\n"
+ "except Exception:\n"
+ " detect_logit_transforms = None\n"
+ "try:\n"
+ " from unsloth_zoo.rl_replacements import distillation_chunked_jsd\n"
+ "except Exception:\n"
+ " distillation_chunked_jsd = None\n"
+)