Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix GKD normalization after response generation (#15222)

cherry77-cloudb39fc8879166
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 09eeff27678..d1e95f22a70 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -180,6 +180,61 @@ def gkd_trainer_mask_prompt(function_name, function):
RL_FUNCTIONS["gkd_trainer"].append(gkd_trainer_mask_prompt)
+def gkd_trainer_generate_before_counting(function_name, function):
+ if function_name != "training_step":
+ return function
+ start = re.search(
+ r"^ if (?:self\.seq_kd|random\.random\(\) <= self\.lmbda):", function, re.M
+ )
+ end = function.find(" loss = super().training_step(")
+ if start is None or end < start.start():
+ return function
+ start = start.start()
+ generation = function[start:end]
+ function = (
+ function[:start]
+ + """ pending = getattr(self, "_unsloth_gkd_pending_batches", None)
+ if pending is not None:
+ batches, batch_args, batch_kwargs = pending
+ self._unsloth_gkd_pending_batches = None
+
+ def prepared_batches():
+ for batch in batches:
+ self._unsloth_prepare_gkd_batch(model, batch)
+ yield batch
+
+ _, count = super().get_batch_samples(
+ prepared_batches(), len(batches), *batch_args, **batch_kwargs
+ )
+ for batch in batches:
+ batch["_unsloth_gkd_num_items"] = count
+
+ if "_unsloth_gkd_num_items" in inputs:
+ num_items_in_batch = inputs.pop("_unsloth_gkd_num_items")
+ else:
+ self._unsloth_prepare_gkd_batch(model, inputs)
+"""
+ + function[end:]
+ )
+ # Prepare the whole accumulation window before counting its replacement
+ # labels. Keep generation in training_step, after checkpoint RNG restoration.
+ return (
+ function
+ + "\n def _unsloth_prepare_gkd_batch(self, model, inputs):\n"
+ + generation
+ + """
+ def get_batch_samples(self, epoch_iterator, num_batches, *args, **kwargs):
+ batches, count = super().get_batch_samples(epoch_iterator, num_batches, *args, **kwargs)
+ if self.lmbda > 0 or self.seq_kd:
+ self._unsloth_gkd_pending_batches = (batches, args, kwargs)
+ return batches, count
+"""
+ )
+
+
+RL_FUNCTIONS["gkd_trainer"].append(gkd_trainer_generate_before_counting)
+
+
def ppo_trainer_free_rollout_logits(function_name, function):
# TRL keeps the float32 (batch, response_length, vocab) generation logits alive through every PPO
# epoch, though only the logprobs taken from them are read: 1.2 GB on a T4 for 8 x 256 Qwen3 tokens.