Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GKD: never pass a zero generated-token count (#15328)

Daniel Han6176ae627492
diff --git a/tests/python/test_gkd_generated_token_count.py b/tests/python/test_gkd_generated_token_count.py
new file mode 100644
index 00000000000..c41c4da12c2
--- /dev/null
+++ b/tests/python/test_gkd_generated_token_count.py
@@ -0,0 +1,240 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""GKD counts num_items_in_batch from the generated labels the loss sees, not the dataset answers."""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import random
+import textwrap
+
+import pytest
+import torch
+
+os.environ.setdefault("UNSLOTH_COMPILE_DISABLE", "1")
+import unsloth.models.rl_replacements as rl
+
+# TRL <= 1.6 runs seq_kd first; TRL >= 1.7 runs the on-policy branch first, seq_kd as its elif.
+SEQ_KD_FIRST = """
+ def training_step(self, model, inputs, num_items_in_batch=None):
+ if self.seq_kd:
+ with unwrap_model_for_generation(self.teacher_model, self.accelerator) as unwrapped_model:
+ new_input_ids, new_attention_mask, new_labels = self.generate_on_policy_outputs(
+ unwrapped_model, inputs, self.generation_config, self.processing_class.pad_token_id
+ )
+ inputs["input_ids"] = new_input_ids
+ inputs["attention_mask"] = new_attention_mask
+ inputs["labels"] = new_labels
+ if random.random() <= self.lmbda:
+ with unwrap_model_for_generation(model, self.accelerator) as unwrapped_model:
+ new_input_ids, new_attention_mask, new_labels = self.generate_on_policy_outputs(
+ unwrapped_model, inputs, self.generation_config, self.processing_class.pad_token_id
+ )
+ inputs["input_ids"] = new_input_ids
+ inputs["attention_mask"] = new_attention_mask
+ inputs["labels"] = new_labels
+
+ loss = super().training_step(model, inputs, num_items_in_batch)
+ return loss
+"""
+LMBDA_FIRST = """
+ def training_step(self, model, inputs, num_items_in_batch=None):
+ if random.random() <= self.lmbda:
+ with unwrap_model_for_generation(model, self.accelerator) as unwrapped_model:
+ new_input_ids, new_attention_mask, new_labels = self.generate_on_policy_outputs(
+ unwrapped_model, inputs, self.generation_config
+ )
+ inputs["input_ids"] = new_input_ids
+ inputs["attention_mask"] = new_attention_mask
+ inputs["labels"] = new_labels
+ elif self.seq_kd:
+ with unwrap_model_for_generation(self.teacher_model, self.accelerator) as unwrapped_model:
+ new_input_ids, new_attention_mask, new_labels = self.generate_on_policy_outputs(
+ unwrapped_model, inputs, self.generation_config
+ )
+ inputs["input_ids"] = new_input_ids
+ inputs["attention_mask"] = new_attention_mask
+ inputs["labels"] = new_labels
+
+ loss = super().training_step(model, inputs, num_items_in_batch)
+ return loss
+"""
+
+GENERATED_LEN = 3
+
+
+class _Parent:
+ def get_batch_samples(
+ self,
+ epoch_iterator,
+ num_batches,
+ device = None,
+ ):
+ batches = [next(epoch_iterator) for _ in range(num_batches)]
+ return batches, sum(int((b["labels"][..., 1:] != -100).sum()) for b in batches)
+
+ def training_step(
+ self,
+ model,
+ inputs,
+ num_items_in_batch = None,
+ ):
+ self.seen.append((num_items_in_batch, sorted(inputs)))
+ return torch.tensor(0.0)
+
+
+class _NoGen:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def __enter__(self):
+ return None
+
+ def __exit__(self, *exc):
+ return False
+
+
+def _build(source, lmbda, seq_kd):
+ rewritten = rl.gkd_trainer_generate_before_counting("training_step", source)
+ namespace = {
+ "random": random,
+ "torch": torch,
+ "unwrap_model_for_generation": _NoGen,
+ "_Parent": _Parent,
+ }
+ exec("class Trainer(_Parent):\n" + rewritten, namespace)
+ trainer = namespace["Trainer"]()
+ trainer.lmbda, trainer.seq_kd, trainer.seen, trainer.generated = lmbda, seq_kd, [], []
+ trainer.teacher_model = trainer.accelerator = trainer.generation_config = None
+ trainer.processing_class = type("P", (), {"pad_token_id": 0})()
+ trainer.generated_len, trainer.mask_generated = GENERATED_LEN, False
+
+ def generate(model, inputs, *args):
+ trainer.generated.append(id(inputs))
+ ids = torch.ones(1, inputs["prompts"].shape[1] + trainer.generated_len, dtype = torch.long)
+ labels = ids.clone()
+ labels[:, : inputs["prompts"].shape[1]] = -100
+ if trainer.mask_generated:
+ labels[:] = -100
+ return ids, torch.ones_like(ids), labels
+
+ trainer.generate_on_policy_outputs = generate
+ return rewritten, trainer
+
+
+def _window(answer_lengths, prompt_len = 4):
+ batches = []
+ for n in answer_lengths:
+ labels = torch.full((1, prompt_len + n), 5)
+ labels[:, :prompt_len] = -100
+ batches.append(
+ {
+ "prompts": torch.ones(1, prompt_len, dtype = torch.long),
+ "input_ids": torch.ones(1, prompt_len + n, dtype = torch.long),
+ "attention_mask": torch.ones(1, prompt_len + n, dtype = torch.long),
+ "labels": labels,
+ }
+ )
+ return batches
+
+
+def _train_window(trainer, answer_lengths):
+ batches, count = trainer.get_batch_samples(iter(_window(answer_lengths)), len(answer_lengths))
+ for batch in batches:
+ trainer.training_step(None, batch, count)
+
+
[email protected]("source", [SEQ_KD_FIRST, LMBDA_FIRST], ids = ["seq_kd_first", "lmbda_first"])
[email protected]("lmbda, seq_kd", [(1.0, False), (0.0, True)])
+def test_generated_labels_set_the_count(source, lmbda, seq_kd):
+ rewritten, trainer = _build(source, lmbda, seq_kd)
+ assert rewritten != source
+ _train_window(trainer, [40, 7, 19])
+ # Each microbatch generated once; the shared count is the generated tokens, whatever the dataset answers were.
+ assert len(trainer.generated) == len(set(trainer.generated)) == 3
+ assert [n for n, _ in trainer.seen] == [3 * GENERATED_LEN] * 3
+ assert all(not any(k.startswith("_unsloth") for k in keys) for _, keys in trainer.seen)
+ # A short final window and a second window recount on their own.
+ trainer.seen.clear()
+ _train_window(trainer, [11])
+ assert [n for n, _ in trainer.seen] == [GENERATED_LEN]
+
+
[email protected]("source", [SEQ_KD_FIRST, LMBDA_FIRST], ids = ["seq_kd_first", "lmbda_first"])
+def test_mixed_window_draws_the_same_random_numbers(source):
+ random.seed(1)
+ expected = [random.random() <= 0.5 for _ in range(6)]
+ random.seed(1)
+ _, trainer = _build(source, 0.5, False)
+ _train_window(trainer, [40, 7, 19])
+ _train_window(trainer, [8, 9, 10])
+ assert len(trainer.generated) == sum(expected)
+ answers = [40, 7, 19, 8, 9, 10]
+ windows = [expected[:3], expected[3:]]
+ totals = [
+ sum(GENERATED_LEN if gen else n for gen, n in zip(flags, answers[i * 3 : i * 3 + 3]))
+ for i, flags in enumerate(windows)
+ ]
+ assert [n for n, _ in trainer.seen] == [totals[0]] * 3 + [totals[1]] * 3
+
+
[email protected]("source", [SEQ_KD_FIRST, LMBDA_FIRST], ids = ["seq_kd_first", "lmbda_first"])
+def test_off_policy_keeps_the_dataset_count(source):
+ _, trainer = _build(source, 0.0, False)
+ _train_window(trainer, [40, 7, 19])
+ assert trainer.generated == []
+ assert [n for n, _ in trainer.seen] == [66] * 3
+
+
+def test_unrecognised_training_step_is_left_alone():
+ source = SEQ_KD_FIRST.replace("loss = super().training_step(", "loss = self._step(")
+ assert rl.gkd_trainer_generate_before_counting("training_step", source) == source
+ assert rl.gkd_trainer_generate_before_counting("compute_loss", SEQ_KD_FIRST) == SEQ_KD_FIRST
+
+
+def test_installed_trl_training_step_is_rewritten():
+ """Read TRL's file, not the class: importing unsloth may have rebound the class to the patched one."""
+ import ast
+
+ for name in ("trl.experimental.gkd.gkd_trainer", "trl.trainer.gkd_trainer"):
+ try:
+ spec = importlib.util.find_spec(name)
+ except Exception:
+ spec = None
+ if spec is None or spec.origin is None:
+ continue
+ with open(spec.origin, encoding = "utf-8") as file:
+ tree = ast.parse(file.read())
+ lines = open(spec.origin, encoding = "utf-8").read().splitlines(keepends = True)
+ for node in ast.walk(tree):
+ if isinstance(node, ast.ClassDef) and node.name == "GKDTrainer":
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef) and item.name == "training_step":
+ source = "".join(lines[item.lineno - 1 : item.end_lineno])
+ rewritten = rl.gkd_trainer_generate_before_counting("training_step", source)
+ assert rewritten != source
+ ast.parse(textwrap.dedent("class T:\n" + rewritten))
+ return
+ pytest.skip("installed trl has no GKDTrainer")
+
+
[email protected]("tensor_count", [False, True])
+def test_all_masked_generations_count_one_not_zero(tensor_count):
+ # TRL 1.7-1.8 mask a pad == eos completion entirely; a zero count would make the JSD sum / 0 NaN.
+ _, trainer = _build(LMBDA_FIRST, 1.0, False)
+ trainer.mask_generated = True
+ if tensor_count:
+ parent = _Parent.get_batch_samples
+
+ def tensor_counts(self, *args, **kwargs):
+ batches, count = parent(self, *args, **kwargs)
+ return batches, torch.tensor(count)
+
+ type(trainer).__bases__[0].get_batch_samples = tensor_counts
+ try:
+ _train_window(trainer, [40, 7])
+ finally:
+ if tensor_count:
+ type(trainer).__bases__[0].get_batch_samples = parent
+ assert [int(n) for n, _ in trainer.seen] == [1, 1]
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index d1e95f22a70..e01bae0be16 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -206,6 +206,11 @@ def gkd_trainer_generate_before_counting(function_name, function):
_, count = super().get_batch_samples(
prepared_batches(), len(batches), *batch_args, **batch_kwargs
)
+ # TRL 1.7-1.8 mask EOS when it is the pad token, so an all-EOS window counts 0: sum / 0 is NaN.
+ if torch.is_tensor(count):
+ count = count.clamp_min(1)
+ elif count is not None:
+ count = max(count, 1)
for batch in batches:
batch["_unsloth_gkd_num_items"] = count